@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d93bf4da85
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Model] Refactoring of MiniCPM-V and add MiniCPM-o-2.6 support for vLLM (#12069)

Signed-off-by: hzh <hezhihui_thu@163.com> Signed-off-by: Sungjae Lee <33976427+llsj14@users.noreply.github.com> Signed-off-by: shaochangxu.scx <shaochangxu.scx@antgroup.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: NickLucche <nlucches@redhat.com> Signed-off-by: Isotr0py <2037008807@qq.com> Signed-off-by: Roger Wang <ywang@roblox.com> Signed-off-by: Rafael Vasquez <rafvasq21@gmail.com> Signed-off-by: Akshat Tripathi <akshat@krai.ai> Signed-off-by: Oleg Mosalov <oleg@krai.ai> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com> Signed-off-by: Yida Wu <yidawu@alumni.cmu.edu> Signed-off-by: Chenguang Li <757486878@qq.com> Signed-off-by: youkaichao <youkaichao@gmail.com> Signed-off-by: Alex-Brooks <Alex.brooks@ibm.com> Signed-off-by: Chen Zhang <zhangch99@outlook.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Shanshan Shen <467638484@qq.com> Signed-off-by: elijah <f1renze.142857@gmail.com> Signed-off-by: Yikun <yikunkero@gmail.com> Signed-off-by: mgoin <michael@neuralmagic.com> Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Signed-off-by: Konrad Zawora <kzawora@habana.ai> Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com> Signed-off-by: Rui Qiao <ruisearch42@gmail.com> Co-authored-by: Sungjae Lee <33976427+llsj14@users.noreply.github.com> Co-authored-by: shaochangxu <85155497+shaochangxu@users.noreply.github.com> Co-authored-by: shaochangxu.scx <shaochangxu.scx@antgroup.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com> Co-authored-by: sixgod <evethwillbeok@outlook.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com> Co-authored-by: Rafael Vasquez <rafvasq21@gmail.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Akshat Tripathi <Akshat.tripathi6568@gmail.com> Co-authored-by: Oleg Mosalov <oleg@krai.ai> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Avshalom Manevich <12231371+avshalomman@users.noreply.github.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-neuralmagic@users.noreply.github.com> Co-authored-by: Yangcheng Li <liyangcheng.lyc@alibaba-inc.com> Co-authored-by: Siyuan Li <94890248+liaoyanqing666@users.noreply.github.com> Co-authored-by: Concurrensee <yida.wu@amd.com> Co-authored-by: Chenguang Li <757486878@qq.com> Co-authored-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Alex Brooks <alex.brooks@ibm.com> Co-authored-by: Chen Zhang <zhangch99@outlook.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Shanshan Shen <467638484@qq.com> Co-authored-by: elijah <30852919+e1ijah1@users.noreply.github.com> Co-authored-by: Yikun Jiang <yikunkero@gmail.com> Co-authored-by: Steve Luo <36296769+SunflowerAries@users.noreply.github.com> Co-authored-by: mgoin <michael@neuralmagic.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: Konrad Zawora <kzawora@habana.ai> Co-authored-by: TJian <tunjian1996@gmail.com> Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: wangxiyuan <wangxiyuan1007@gmail.com> Co-authored-by: maang-h <55082429+maang-h@users.noreply.github.com> Co-authored-by: Elfie Guo <164945471+elfiegg@users.noreply.github.com> Co-authored-by: Rui Qiao <161574667+ruisearch42@users.noreply.github.com> Co-authored-by: Roger Wang <ywang@roblox.com>

Alphi committed Jan 29, 2025 at 17:24 UTC d93bf4da855a0c5e8d3c875def6b37c5e9d77763
15 files changed +1622 -186
docs/source/models/supported_models.md
+8 -1
@@ -693,9 +693,16 @@ See [this page](#generative-models) for more information on how to use generativ
693 *
694 * ✅︎
695 * ✅︎
696 +- * `MiniCPMO`
697 + * MiniCPM-O
698 + * T + I<sup>E+</sup> + V<sup>E+</sup> + A<sup>E+</sup>
699 + * `openbmb/MiniCPM-o-2_6`, etc.
700 + * ✅︎
701 + * ✅︎
702 + *
703 - * `MiniCPMV`
704 * MiniCPM-V
698 - * T + I<sup>E+</sup>
705 + * T + I<sup>E+</sup> + V<sup>E+</sup>
706 * `openbmb/MiniCPM-V-2` (see note), `openbmb/MiniCPM-Llama3-V-2_5`, `openbmb/MiniCPM-V-2_6`, etc.
707 * ✅︎
708 * ✅︎
examples/offline_inference/audio_language.py
+31 -1
@@ -67,7 +67,37 @@ def run_qwen2_audio(question: str, audio_count: int):
67 return llm, prompt, stop_token_ids
68
69
70 -model_example_map = {"ultravox": run_ultravox, "qwen2_audio": run_qwen2_audio}
70 +def run_minicpmo(question: str, audio_count: int):
71 + model_name = "openbmb/MiniCPM-o-2_6"
72 + tokenizer = AutoTokenizer.from_pretrained(model_name,
73 + trust_remote_code=True)
74 + llm = LLM(model=model_name,
75 + trust_remote_code=True,
76 + max_model_len=4096,
77 + max_num_seqs=5,
78 + limit_mm_per_prompt={"audio": audio_count})
79 +
80 + stop_tokens = ['<|im_end|>', '<|endoftext|>']
81 + stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
82 +
83 + audio_placeholder = "(<audio>./</audio>)" * audio_count
84 + audio_chat_template = "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n<|spk_bos|><|spk|><|spk_eos|><|tts_bos|>' }}{% endif %}" # noqa: E501
85 + messages = [{
86 + 'role': 'user',
87 + 'content': f'{audio_placeholder}\n{question}'
88 + }]
89 + prompt = tokenizer.apply_chat_template(messages,
90 + tokenize=False,
91 + add_generation_prompt=True,
92 + chat_template=audio_chat_template)
93 + return llm, prompt, stop_token_ids
94 +
95 +
96 +model_example_map = {
97 + "ultravox": run_ultravox,
98 + "qwen2_audio": run_qwen2_audio,
99 + "minicpmo": run_minicpmo
100 +}
101
102
103 def main(args):
examples/offline_inference/vision_language.py
+28 -5
@@ -265,8 +265,9 @@ def run_mantis(question: str, modality: str):
265
266
267 # MiniCPM-V
268 -def run_minicpmv(question: str, modality: str):
269 - assert modality == "image"
268 +def run_minicpmv_base(question: str, modality: str, model_name):
269 + assert modality in ["image", "video"]
270 + # If you want to use `MiniCPM-o-2_6` with audio inputs, check `audio_language.py` # noqa
271
272 # 2.0
273 # The official repo doesn't work yet, so we need to use a fork for now
@@ -277,7 +278,15 @@ def run_minicpmv(question: str, modality: str):
278 # model_name = "openbmb/MiniCPM-Llama3-V-2_5"
279
280 # 2.6
280 - model_name = "openbmb/MiniCPM-V-2_6"
281 + # model_name = "openbmb/MiniCPM-V-2_6"
282 + # o2.6
283 +
284 + # modality supports
285 + # 2.0: image
286 + # 2.5: image
287 + # 2.6: image, video
288 + # o2.6: image, video, audio
289 + # model_name = "openbmb/MiniCPM-o-2_6"
290 tokenizer = AutoTokenizer.from_pretrained(model_name,
291 trust_remote_code=True)
292 llm = LLM(
@@ -294,13 +303,18 @@ def run_minicpmv(question: str, modality: str):
303 # 2.5
304 # stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]
305
297 - # 2.6
306 + # 2.6 / o2.6
307 stop_tokens = ['<|im_end|>', '<|endoftext|>']
308 stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
309
310 + modality_placeholder = {
311 + "image": "(<image>./</image>)",
312 + "video": "(<video>./</video>)",
313 + }
314 +
315 messages = [{
316 'role': 'user',
303 - 'content': f'(<image>./</image>)\n{question}'
317 + 'content': f'{modality_placeholder[modality]}\n{question}'
318 }]
319 prompt = tokenizer.apply_chat_template(messages,
320 tokenize=False,
@@ -308,6 +322,14 @@ def run_minicpmv(question: str, modality: str):
322 return llm, prompt, stop_token_ids
323
324
325 +def run_minicpmo(question: str, modality: str):
326 + return run_minicpmv_base(question, modality, "openbmb/MiniCPM-o-2_6")
327 +
328 +
329 +def run_minicpmv(question: str, modality: str):
330 + return run_minicpmv_base(question, modality, "openbmb/MiniCPM-V-2_6")
331 +
332 +
333 # LLama 3.2
334 def run_mllama(question: str, modality: str):
335 assert modality == "image"
@@ -523,6 +545,7 @@ model_example_map = {
545 "llava-next-video": run_llava_next_video,
546 "llava-onevision": run_llava_onevision,
547 "mantis": run_mantis,
548 + "minicpmo": run_minicpmo,
549 "minicpmv": run_minicpmv,
550 "mllama": run_mllama,
551 "molmo": run_molmo,
requirements-cpu.txt
+1
@@ -4,5 +4,6 @@
4 # Dependencies for CPUs
5 torch==2.5.1+cpu; platform_machine != "ppc64le" and platform_machine != "aarch64" and platform_system != "Darwin"
6 torch==2.5.1; platform_machine == "aarch64" or platform_system == "Darwin"
7 +torchaudio; platform_machine != "ppc64le" # required for the image processor of minicpm-o-2_6, this must be updated alongside torch
8 torchvision; platform_machine != "ppc64le" # required for the image processor of phi3v, this must be updated alongside torch
9 datasets # for benchmark scripts
requirements-cuda.txt
+1
@@ -5,6 +5,7 @@
5 ray[default] >= 2.9
6 nvidia-ml-py >= 12.560.30 # for pynvml package
7 torch == 2.5.1
8 +torchaudio==2.5.1
9 # These must be updated alongside torch
10 torchvision == 0.20.1 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
11 xformers == 0.0.28.post3; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch 2.5.1
requirements-test.in
+3
@@ -12,6 +12,8 @@ decord # required for video tests
12 einops # required for MPT, qwen-vl and Mamba
13 httpx
14 librosa # required for audio tests
15 +vector_quantize_pytorch # required for minicpmo_26 test
16 +vocos # required for minicpmo_26 test
17 peft
18 pqdm
19 ray[adag]==2.40.0
@@ -19,6 +21,7 @@ sentence-transformers # required for embedding tests
21 soundfile # required for audio tests
22 timm # required for internvl test
23 torch==2.5.1
24 +torchaudio==2.5.1
25 transformers_stream_generator # required for qwen-vl test
26 matplotlib # required for qwen-vl test
27 mistral_common[opencv] >= 1.5.0 # required for pixtral test
requirements-test.txt
+35 -2
@@ -106,9 +106,17 @@ dnspython==2.7.0
106 docutils==0.16
107 # via awscli
108 einops==0.8.0
109 - # via -r requirements-test.in
109 + # via
110 + # -r requirements-test.in
111 + # encodec
112 + # vector-quantize-pytorch
113 + # vocos
114 +einx==0.3.0
115 + # via vector-quantize-pytorch
116 email-validator==2.2.0
117 # via pydantic
118 +encodec==0.1.1
119 + # via vocos
120 evaluate==0.4.3
121 # via lm-eval
122 fastparquet==2024.11.0
@@ -125,6 +133,8 @@ filelock==3.16.1
133 # triton
134 fonttools==4.54.1
135 # via matplotlib
136 +frozendict==2.4.6
137 + # via einx
138 frozenlist==1.5.0
139 # via
140 # aiohttp
@@ -159,6 +169,7 @@ huggingface-hub==0.26.2
169 # timm
170 # tokenizers
171 # transformers
172 + # vocos
173 idna==3.10
174 # via
175 # anyio
@@ -261,6 +272,8 @@ numpy==1.26.4
272 # cupy-cuda12x
273 # datasets
274 # decord
275 + # einx
276 + # encodec
277 # evaluate
278 # fastparquet
279 # genai-perf
@@ -283,6 +296,7 @@ numpy==1.26.4
296 # torchvision
297 # transformers
298 # tritonclient
299 + # vocos
300 nvidia-cublas-cu12==12.4.5.8
301 # via
302 # nvidia-cudnn-cu12
@@ -455,6 +469,7 @@ pyyaml==6.0.2
469 # responses
470 # timm
471 # transformers
472 + # vocos
473 ray[adag]==2.40.0
474 # via -r requirements-test.in
475 redis==5.2.0
@@ -517,6 +532,7 @@ scipy==1.13.1
532 # scikit-learn
533 # sentence-transformers
534 # statsmodels
535 + # vocos
536 sentence-transformers==3.2.1
537 # via -r requirements-test.in
538 sentencepiece==0.2.0
@@ -540,7 +556,9 @@ sqlitedict==2.1.0
556 statsmodels==0.14.4
557 # via genai-perf
558 sympy==1.13.1
543 - # via torch
559 + # via
560 + # einx
561 + # torch
562 tabledata==1.3.3
563 # via pytablewriter
564 tabulate==0.9.0
@@ -568,12 +586,21 @@ torch==2.5.1
586 # -r requirements-test.in
587 # accelerate
588 # bitsandbytes
589 + # encodec
590 # lm-eval
591 # peft
592 # sentence-transformers
593 # tensorizer
594 # timm
595 + # torchaudio
596 # torchvision
597 + # vector-quantize-pytorch
598 + # vocos
599 +torchaudio==2.5.1
600 + # via
601 + # -r requirements-test.in
602 + # encodec
603 + # vocos
604 torchvision==0.20.1
605 # via timm
606 tqdm==4.66.6
@@ -584,6 +611,7 @@ tqdm==4.66.6
611 # lm-eval
612 # nltk
613 # peft
614 + # pqdm
615 # sentence-transformers
616 # tqdm-multiprocess
617 # transformers
@@ -615,6 +643,7 @@ typing-extensions==4.12.2
643 # huggingface-hub
644 # librosa
645 # mistral-common
646 + # pqdm
647 # pydantic
648 # pydantic-core
649 # torch
@@ -626,6 +655,10 @@ urllib3==2.2.3
655 # requests
656 # responses
657 # tritonclient
658 +vector-quantize-pytorch==1.21.2
659 + # via -r requirements-test.in
660 +vocos==0.1.0
661 + # via -r requirements-test.in
662 word2number==1.1
663 # via lm-eval
664 xxhash==3.5.0
tests/models/decoder_only/vision_language/test_models.py
+14
@@ -350,6 +350,20 @@ VLM_TEST_SETTINGS = {
350 postprocess_inputs=model_utils.wrap_inputs_post_processor,
351 hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
352 ),
353 + "minicpmo_26": VLMTestInfo(
354 + models=["openbmb/MiniCPM-o-2_6"],
355 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
356 + prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
357 + img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",
358 + max_model_len=4096,
359 + max_num_seqs=2,
360 + get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(['<|im_end|>', '<|endoftext|>']), # noqa: E501
361 + postprocess_inputs=model_utils.ignore_inputs_post_processor(
362 + "image_sizes"
363 + ),
364 + hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
365 + patch_hf_runner=model_utils.minicpmo_patch_hf_runner
366 + ),
367 "minicpmv_26": VLMTestInfo(
368 models=["openbmb/MiniCPM-V-2_6"],
369 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+11
@@ -497,6 +497,17 @@ def mantis_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
497 return hf_model
498
499
500 +def minicpmo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
501 + orig_generate = hf_model.model.generate
502 +
503 + def _generate(self, *args, **kwargs):
504 + return orig_generate(*args, decode_text=False, **kwargs)
505 +
506 + hf_model.model.generate = types.MethodType(_generate, hf_model.model)
507 +
508 + return hf_model
509 +
510 +
511 def _generate_greedy_logprobs_limit(
512 self,
513 prompts: List[str],
tests/models/multimodal/processing/test_common.py
+2
@@ -152,6 +152,8 @@ def _test_processing_correctness(
152 "llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
153 "TIGER-Lab/Mantis-8B-siglip-llama3",
154 "mistral-community/pixtral-12b",
155 + "openbmb/MiniCPM-o-2_6",
156 + "openbmb/MiniCPM-V-2_6",
157 "Qwen/Qwen-VL-Chat",
158 "Qwen/Qwen2-VL-2B-Instruct",
159 "Qwen/Qwen2-Audio-7B-Instruct",
tests/models/registry.py
+3 -1
@@ -245,7 +245,9 @@ _MULTIMODAL_EXAMPLE_MODELS = {
245 "LlavaOnevisionForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-onevision-qwen2-0.5b-ov-hf"), # noqa: E501
246 "MantisForConditionalGeneration": _HfExamplesInfo("TIGER-Lab/Mantis-8B-siglip-llama3", # noqa: E501
247 hf_overrides={"architectures": ["MantisForConditionalGeneration"]}), # noqa: E501
248 - "MiniCPMV": _HfExamplesInfo("openbmb/MiniCPM-Llama3-V-2_5",
248 + "MiniCPMO": _HfExamplesInfo("openbmb/MiniCPM-o-2_6",
249 + trust_remote_code=True),
250 + "MiniCPMV": _HfExamplesInfo("openbmb/MiniCPM-V-2_6",
251 trust_remote_code=True),
252 "MolmoForCausalLM": _HfExamplesInfo("allenai/Molmo-7B-D-0924",
253 trust_remote_code=True),
vllm/entrypoints/chat_utils.py
+5 -1
@@ -392,7 +392,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
392 if model_type == "phi3_v":
393 # Workaround since this token is not defined in the tokenizer
394 return f"<|image_{current_count}|>"
395 - if model_type == "minicpmv":
395 + if model_type in ("minicpmo", "minicpmv"):
396 return "(<image>./</image>)"
397 if model_type in ("blip-2", "chatglm", "fuyu", "paligemma",
398 "pixtral"):
@@ -424,10 +424,14 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
424 if model_type == "qwen2_audio":
425 return (f"Audio {current_count}: "
426 f"<|audio_bos|><|AUDIO|><|audio_eos|>")
427 + if model_type == "minicpmo":
428 + return "(<audio>./</audio>)"
429 raise TypeError(f"Unknown model type: {model_type}")
430 elif modality == "video":
431 if model_type == "qwen2_vl":
432 return "<|vision_start|><|video_pad|><|vision_end|>"
433 + if model_type in ("minicpmo", "minicpmv"):
434 + return "(<video>./</video>)"
435 if model_type.startswith("llava"):
436 return self._cached_token_str(self._tokenizer,
437 hf_config.video_token_index)
vllm/model_executor/models/minicpmo.py new
+811
@@ -0,0 +1,811 @@
1 +# Adapted from
2 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 +# Copyright 2023 The vLLM team.
4 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
5 +#
6 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
7 +# and OPT implementations in this library. It has been modified from its
8 +# original forms to accommodate minor architectural differences compared
9 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
10 +#
11 +# Licensed under the Apache License, Version 2.0 (the "License");
12 +# you may not use this file except in compliance with the License.
13 +# You may obtain a copy of the License at
14 +#
15 +# http://www.apache.org/licenses/LICENSE-2.0
16 +#
17 +# Unless required by applicable law or agreed to in writing, software
18 +# distributed under the License is distributed on an "AS IS" BASIS,
19 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20 +# See the License for the specific language governing permissions and
21 +# limitations under the License.
22 +"""Inference-only MiniCPM-O model compatible with HuggingFace weights."""
23 +from functools import partial
24 +from itertools import accumulate
25 +from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, Set,
26 + Tuple, TypedDict, Union)
27 +
28 +import torch
29 +import torch.types
30 +from torch import nn
31 +from transformers.modeling_outputs import BaseModelOutputWithPast
32 +from transformers.models.whisper.modeling_whisper import (
33 + ACT2FN, WHISPER_ATTENTION_CLASSES, WhisperConfig, WhisperEncoder)
34 +
35 +from vllm.attention import AttentionMetadata
36 +from vllm.config import VllmConfig
37 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
38 +from vllm.multimodal.inputs import MultiModalFieldConfig
39 +from vllm.multimodal.parse import (ModalityData, ModalityDataItems,
40 + MultiModalDataItems, MultiModalDataParser,
41 + VideoItem)
42 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
43 + PromptReplacement)
44 +from vllm.multimodal.profiling import ProcessorInputs
45 +from vllm.sequence import IntermediateTensors
46 +
47 +from .minicpmv import (MiniCPMV2_6, MiniCPMVDummyInputsBuilder,
48 + MiniCPMVEmbeddingItems, MiniCPMVMultiModalDataParser,
49 + MiniCPMVMultiModalProcessor, MiniCPMVProcessingInfo)
50 +from .utils import AutoWeightsLoader, maybe_prefix
51 +
52 +CPU_DEVICE = torch.device("cpu")
53 +
54 +MiniCPMOEmbeddingItems = MiniCPMVEmbeddingItems
55 +
56 +
57 +class MiniCPMOAudioFeatureInputs(TypedDict):
58 + type: Literal["audio_features"]
59 + data: torch.Tensor
60 + """
61 + Shape: `(batch_size * num_audios * num_slices, num_channels, length)`
62 + Slice here means chunk. Audio that is too long will be split into slices,
63 + which is the same as image.
64 + Padding is used therefore `data` is `torch.Tensor`.
65 + """
66 +
67 + audio_feature_lens: torch.Tensor
68 + """
69 + Shape: `(batch_size * num_audios * num_slices)`
70 +
71 + This should be feature length of each audio slice,
72 + which equals to `data.shape[-1]`
73 + """
74 +
75 + audio_bounds: torch.Tensor
76 + """
77 + Shape: `(batch_size * num_audios * num_slices, 2)`
78 +
79 + This should be in `(start, stop)` format.
80 + """
81 +
82 +
83 +class MiniCPMOAudioEmbeddingInputs(TypedDict):
84 + type: Literal["audio_embeds"]
85 + data: List[torch.Tensor]
86 + """
87 + Shape: `(batch_size * num_images * num_slices, hidden_size)`
88 +
89 + `hidden_size` must match the hidden size of language model backbone.
90 + instead of a batched tensor.
91 + Length of each slice may vary, so pass it as a list.
92 + """
93 + audio_bounds: torch.Tensor
94 + """
95 + Shape: `(batch_size * num_audios * num_slices, 2)`
96 +
97 + This should be in `(start, stop)` format.
98 + """
99 +
100 +
101 +MiniCPMOAudioInputs = Union[MiniCPMOAudioFeatureInputs,
102 + MiniCPMOAudioEmbeddingInputs]
103 +
104 +
105 +class MiniCPMOAudioEmbeddingItems(MiniCPMOEmbeddingItems):
106 +
107 + def __init__(self, data: Dict) -> None:
108 + super().__init__(data, "audio")
109 + audio_embeds = self.data.get("audio_embeds", None)
110 + if audio_embeds is None:
111 + raise ValueError("Incorrect type of video_embeds",
112 + "Got type: None")
113 + self.data["audio_embeds"] = audio_embeds
114 +
115 + def get(self, index: int) -> object:
116 + return self.data["audio_embeds"][index]
117 +
118 +
119 +class MiniCPMOMultiModalDataParser(MiniCPMVMultiModalDataParser):
120 +
121 + def _parse_audio_data(
122 + self,
123 + data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],
124 + ) -> ModalityDataItems[Any, Any]:
125 + if isinstance(data, dict):
126 + return MiniCPMOAudioEmbeddingItems(data)
127 + return super()._parse_audio_data(data)
128 +
129 +
130 +class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
131 + audio_pattern = "(<audio>./</audio>)"
132 +
133 + def get_supported_mm_modalities(self) -> List[str]:
134 + return ["image", "video", "audio"]
135 +
136 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
137 + return {"image": None, "video": None, "audio": None}
138 +
139 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
140 + return {
141 + "image": self.get_max_image_tokens(),
142 + "audio": self.get_max_audio_tokens(),
143 + "video": self.get_max_video_tokens(seq_len)
144 + }
145 +
146 + def get_default_audio_pool_step(self) -> int:
147 + return 2
148 +
149 + def get_default_audio_sampling_rate(self) -> int:
150 + return 16000
151 +
152 + def get_chunk_length(self) -> int:
153 + return self.get_hf_config().audio_chunk_length
154 +
155 + def get_max_audio_tokens_per_chunk(self) -> int:
156 + pool_step = self.get_default_audio_pool_step()
157 + fbank_feat_in_chunk = 100
158 + cnn_feat_in_chunk = (fbank_feat_in_chunk - 1) // 2 + 1
159 + num_audio_tokens = (cnn_feat_in_chunk - pool_step) // pool_step + 1
160 + return num_audio_tokens + 2 # <audio>(<unk>*N)</audio>
161 +
162 + def get_max_audio_chunks_with_most_features(self) -> int:
163 + return 30
164 +
165 + def get_audio_len_by_num_chunks(self, num_chunks: int) -> int:
166 + sampling_rate = self.get_default_audio_sampling_rate()
167 + # exclude <audio> </audio>
168 + num_tokens_per_chunk = self.get_max_audio_tokens_per_chunk() - 2
169 + return int(num_chunks * sampling_rate / num_tokens_per_chunk) + 1
170 +
171 + def get_num_frames_with_most_features(self, seq_len: int) -> int:
172 + mm_config = self.ctx.get_mm_config()
173 + max_images = mm_config.limit_per_prompt.get("image", 1)
174 + max_videos = mm_config.limit_per_prompt.get("video", 1)
175 + max_audios = mm_config.limit_per_prompt.get("audio", 1)
176 +
177 + # count <image_idx></image_idx> tokens
178 + # which are not in get_max_image_tokens
179 + max_image_tokens = self.get_max_image_tokens(
180 + ) * max_images + 4 * max_images
181 + max_audio_tokens = self.get_max_audio_tokens(
182 + ) * max_audios + 2 * max_audios
183 + max_total_frames = self.get_max_video_frames(seq_len -
184 + max_image_tokens -
185 + max_audio_tokens)
186 +
187 + num_frames = max(max_total_frames // max(max_videos, 1), 1)
188 +
189 + return num_frames
190 +
191 +
192 +class MiniCPMODummyInputsBuilder(MiniCPMVDummyInputsBuilder):
193 +
194 + def get_dummy_processor_inputs(
195 + self, seq_len: int, mm_counts: Mapping[str,
196 + int]) -> ProcessorInputs:
197 + num_audios = mm_counts.get("audio", 0)
198 + audio_len = self.info.get_max_audio_chunks_with_most_features() * \
199 + self.info.get_default_audio_sampling_rate()
200 +
201 + processor_inputs = super().get_dummy_processor_inputs(
202 + seq_len, mm_counts)
203 + mm_data = {
204 + "image":
205 + processor_inputs.mm_data["image"],
206 + "video":
207 + processor_inputs.mm_data["video"],
208 + "audio":
209 + self._get_dummy_audios(length=audio_len, num_audios=num_audios)
210 + }
211 +
212 + audio_prompt_texts = self.info.audio_pattern * num_audios
213 +
214 + return ProcessorInputs(prompt_text=processor_inputs.prompt_text + \
215 + audio_prompt_texts,
216 + mm_data=mm_data)
217 +
218 +
219 +class MiniCPMOMultiModalProcessor(
220 + MiniCPMVMultiModalProcessor,
221 + BaseMultiModalProcessor[MiniCPMOProcessingInfo]):
222 +
223 + def _get_data_parser(self) -> MultiModalDataParser:
224 + return MiniCPMOMultiModalDataParser(
225 + target_sr=self.info.get_default_audio_sampling_rate())
226 +
227 + def get_audio_prompt_texts(self,
228 + audio_lens: int,
229 + chunk_input: bool = True,
230 + chunk_length: int = 1) -> str:
231 + return self.info.get_hf_processor().get_audio_placeholder(
232 + audio_lens, chunk_input, chunk_length)
233 +
234 + def get_special_tokens(self) -> Dict[str, torch.Tensor]:
235 + tokenizer = self.info.get_tokenizer()
236 + special_tokens = super().get_special_tokens()
237 + if hasattr(tokenizer, "audio_start_id"):
238 + special_tokens["audio_start_id"] = torch.tensor(
239 + tokenizer.audio_start_id)
240 + special_tokens["audio_end_id"] = torch.tensor(
241 + tokenizer.audio_end_id)
242 + return special_tokens
243 +
244 + def process_audios(self, mm_data: Mapping[str, object],
245 + mm_kwargs: Mapping[str, object]) -> Dict[str, object]:
246 + audios = mm_data.pop("audios", [])
247 + audio_embeds = mm_data.pop("audio_embeds", [])
248 + if isinstance(audios, (list, torch.Tensor)) and len(audios) > 0:
249 + audio_outputs = {
250 + "audio_lens": [],
251 + "audio_features": [],
252 + "audio_feature_lens": [],
253 + "audio_num_segments": []
254 + }
255 + for audio in audios:
256 + single_audio_outputs = super().call_base_hf_processor(
257 + prompt=self.info.audio_pattern,
258 + mm_data={
259 + "audios": audio,
260 + "chunk_input": True
261 + },
262 + mm_kwargs=mm_kwargs)
263 + audio_outputs["audio_lens"].append(len(audio))
264 + audio_outputs["audio_features"].append(
265 + single_audio_outputs["audio_features"])
266 + audio_outputs["audio_num_segments"].append(
267 + len(single_audio_outputs["audio_feature_lens"][0]))
268 + audio_outputs["audio_feature_lens"] += \
269 + single_audio_outputs["audio_feature_lens"]
270 + audio_outputs["audio_features"] = [
271 + audio_feature for single_audio_features in \
272 + audio_outputs["audio_features"]
273 + for audio_feature in single_audio_features
274 + ]
275 + audio_outputs["audio_feature_lens"] = torch.cat(
276 + audio_outputs["audio_feature_lens"])
277 + elif len(audio_embeds):
278 + audio_outputs = {
279 + "audio_lens": [
280 + self.info.get_audio_len_by_num_chunks(
281 + sum(chunk_embeds.shape[0]
282 + for chunk_embeds in single_audio_embeds))
283 + for single_audio_embeds in audio_embeds
284 + ],
285 + "audio_embeds": [
286 + chunk_embeds for single_audio_embeds in audio_embeds
287 + for chunk_embeds in single_audio_embeds
288 + ],
289 + "audio_num_segments": [
290 + len(single_audio_embeds)
291 + for single_audio_embeds in audio_embeds
292 + ]
293 + }
294 + else:
295 + audio_outputs = {}
296 + return audio_outputs
297 +
298 + def get_placeholder_match_pattern(self) -> str:
299 + return r"\(<(image|video|audio)>./</\1>\)"
300 +
301 + def get_placeholder_split_pattern(self) -> str:
302 + return r"\(<(?:image|video|audio)>./</(?:image|video|audio)>\)"
303 +
304 + def process_mm_inputs(self, mm_data, mm_kwargs) -> object:
305 + return {
306 + "image": self.process_images(mm_data, mm_kwargs),
307 + "video": self.process_videos(mm_data, mm_kwargs),
308 + "audio": self.process_audios(mm_data, mm_kwargs)
309 + }
310 +
311 + def get_modality_num_counter(self, modality: str) -> str:
312 + if modality == "audio":
313 + return "audio_lens"
314 + return super().get_modality_num_counter(modality)
315 +
316 + def get_num_slices_by_modality(self, inputs: Dict[str, object],
317 + modality: str, index: int) -> int:
318 + if modality == "audio":
319 + return inputs["audio"]["audio_num_segments"][index]
320 + return super().get_num_slices_by_modality(inputs, modality, index)
321 +
322 + def get_prompt_texts_by_modality(self, inputs: Dict[str, object],
323 + modality: str, index: int) -> str:
324 + if modality == "audio":
325 + return self.get_audio_prompt_texts(
326 + inputs["audio"]["audio_lens"][index])
327 + return super().get_prompt_texts_by_modality(inputs, modality, index)
328 +
329 + def _get_prompt_replacements(
330 + self, mm_items: MultiModalDataItems,
331 + hf_processor_mm_kwargs: Mapping[str, Any],
332 + out_mm_kwargs: MultiModalKwargs) -> List[PromptReplacement]:
333 + placeholder = {
334 + "image": self.info.image_pattern,
335 + "video": self.info.video_pattern,
336 + "audio": self.info.audio_pattern
337 + }
338 +
339 + def get_replacement_minicpmv(item_idx: int, modality: str):
340 + if modality == "image":
341 + return self.get_image_prompt_texts(
342 + mm_items["image"].get_image_size(item_idx), item_idx)
343 + elif modality == "video":
344 + return self.get_video_prompt_texts(
345 + mm_items["video"].get_frame_size(item_idx),
346 + mm_items["video"].get_num_frames(item_idx))
347 + else: # audio
348 + if isinstance(mm_items["audio"], MiniCPMOAudioEmbeddingItems):
349 + single_audio_embeds = mm_items["audio"].get(item_idx)
350 + audio_len = self.info.get_audio_len_by_num_chunks(
351 + sum(chunk_embeds.shape[0]
352 + for chunk_embeds in single_audio_embeds))
353 + return self.get_audio_prompt_texts(audio_len)
354 + return self.get_audio_prompt_texts(
355 + len(mm_items["audio"].get(item_idx)))
356 +
357 + return [
358 + PromptReplacement(modality=modality,
359 + target=placeholder[modality],
360 + replacement=partial(get_replacement_minicpmv,
361 + modality=modality))
362 + for modality in ("image", "video", "audio")
363 + ]
364 +
365 + def _get_mm_fields_config(
366 + self,
367 + hf_inputs,
368 + hf_processor_mm_kwargs: Mapping[str, object],
369 + ) -> Mapping[str, MultiModalFieldConfig]:
370 +
371 + def get_slices(num_slices: List[int]) -> List[int]:
372 + slice_indices = [0] + list(accumulate(num_slices))
373 + slices = [(slice_indices[i], slice_indices[i + 1])
374 + for i in range(len(num_slices))]
375 + return [slice(*slice_item) for slice_item in slices]
376 +
377 + audio_slices = get_slices(
378 + hf_inputs.get("audio_num_slices", torch.empty(0)))
379 + return dict(
380 + **super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs),
381 + audio_features=MultiModalFieldConfig.flat("audio", audio_slices),
382 + audio_feature_lens=MultiModalFieldConfig.flat(
383 + "audio", audio_slices),
384 + audio_num_slices=MultiModalFieldConfig.batched("audio"),
385 + audio_orders_in_mm_data=MultiModalFieldConfig.batched("audio"),
386 + audio_embeds=MultiModalFieldConfig.flat("audio", audio_slices))
387 +
388 +
389 +class MultiModalProjector(nn.Module):
390 +
391 + def __init__(self, in_dim: int, out_dim: int):
392 + super().__init__()
393 + self.linear1 = nn.Linear(in_features=in_dim,
394 + out_features=out_dim,
395 + bias=True)
396 + self.relu = nn.ReLU()
397 + self.linear2 = nn.Linear(in_features=out_dim,
398 + out_features=out_dim,
399 + bias=True)
400 +
401 + def forward(self, audio_features: torch.Tensor) -> torch.Tensor:
402 + hidden_states = self.relu(self.linear1(audio_features))
403 + hidden_states = self.linear2(hidden_states)
404 + return hidden_states
405 +
406 +
407 +class MiniCPMWhisperEncoderLayer(nn.Module):
408 +
409 + def __init__(self, config: WhisperConfig, layer_idx: int = None):
410 + super().__init__()
411 + self.embed_dim = config.d_model
412 + self.self_attn = WHISPER_ATTENTION_CLASSES[
413 + config._attn_implementation](
414 + embed_dim=self.embed_dim,
415 + num_heads=config.encoder_attention_heads,
416 + dropout=config.attention_dropout,
417 + config=config,
418 + layer_idx=layer_idx,
419 + )
420 + self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
421 + self.dropout = config.dropout
422 + self.activation_fn = ACT2FN[config.activation_function]
423 + self.activation_dropout = config.activation_dropout
424 + self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim)
425 + self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim)
426 + self.final_layer_norm = nn.LayerNorm(self.embed_dim)
427 +
428 + def forward(
429 + self,
430 + hidden_states: torch.Tensor,
431 + attention_mask: torch.Tensor,
432 + ) -> torch.Tensor:
433 + residual = hidden_states
434 + past_key_values = None
435 + hidden_states = self.self_attn_layer_norm(hidden_states)
436 + hidden_states, attn_weights, past_key_values = self.self_attn(
437 + hidden_states=hidden_states,
438 + attention_mask=attention_mask,
439 + past_key_value=past_key_values,
440 + )
441 + hidden_states = nn.functional.dropout(hidden_states,
442 + p=self.dropout,
443 + training=self.training)
444 + hidden_states = residual + hidden_states
445 +
446 + residual = hidden_states
447 + hidden_states = self.final_layer_norm(hidden_states)
448 + hidden_states = self.activation_fn(self.fc1(hidden_states))
449 + hidden_states = nn.functional.dropout(hidden_states,
450 + p=self.activation_dropout,
451 + training=self.training)
452 + hidden_states = self.fc2(hidden_states)
453 + hidden_states = nn.functional.dropout(hidden_states,
454 + p=self.dropout,
455 + training=self.training)
456 + hidden_states = residual + hidden_states
457 +
458 + if hidden_states.dtype == torch.float16 and (
459 + torch.isinf(hidden_states).any()
460 + or torch.isnan(hidden_states).any()):
461 + clamp_value = torch.finfo(hidden_states.dtype).max - 1000
462 + hidden_states = torch.clamp(hidden_states,
463 + min=-clamp_value,
464 + max=clamp_value)
465 +
466 + outputs = (hidden_states, )
467 +
468 + return outputs
469 +
470 +
471 +class MiniCPMWhisperEncoder(WhisperEncoder):
472 +
473 + def __init__(self, config: WhisperConfig):
474 + super().__init__(config)
475 + self.layers = nn.ModuleList([
476 + MiniCPMWhisperEncoderLayer(config, layer_idx=i)
477 + for i in range(config.encoder_layers)
478 + ])
479 +
480 + def forward(
481 + self,
482 + input_features: torch.Tensor,
483 + attention_mask: Optional[torch.Tensor] = None,
484 + ) -> BaseModelOutputWithPast:
485 + # Ignore copy
486 + input_features = input_features.to(dtype=self.conv1.weight.dtype,
487 + device=self.conv1.weight.device)
488 +
489 + inputs_embeds = nn.functional.gelu(self.conv1(input_features))
490 + inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds))
491 +
492 + inputs_embeds = inputs_embeds.permute(0, 2, 1)
493 +
494 + embed_pos = self.embed_positions.weight
495 +
496 + embed_pos = embed_pos[:inputs_embeds.shape[1], :]
497 +
498 + hidden_states = inputs_embeds + embed_pos
499 + hidden_states = nn.functional.dropout(hidden_states,
500 + p=self.dropout,
501 + training=self.training)
502 +
503 + encoder_states = ()
504 +
505 + for idx, encoder_layer in enumerate(self.layers):
506 + encoder_states = encoder_states + (hidden_states, )
507 + to_drop = False
508 + if self.training:
509 + dropout_probability = torch.rand([])
510 + if dropout_probability < self.layerdrop: # skip the layer
511 + to_drop = True
512 +
513 + # Ignore copy
514 + if to_drop:
515 + layer_outputs = (None, None)
516 + else:
517 + layer_outputs = encoder_layer(
518 + hidden_states,
519 + attention_mask,
520 + )
521 +
522 + hidden_states = layer_outputs[0]
523 +
524 + hidden_states = self.layer_norm(hidden_states)
525 + encoder_states = encoder_states + (hidden_states, )
526 +
527 + return BaseModelOutputWithPast(
528 + last_hidden_state=hidden_states,
529 + hidden_states=encoder_states,
530 + )
531 +
532 +
533 +@MULTIMODAL_REGISTRY.register_processor(
534 + MiniCPMOMultiModalProcessor,
535 + info=MiniCPMOProcessingInfo,
536 + dummy_inputs=MiniCPMODummyInputsBuilder)
537 +class MiniCPMO(MiniCPMV2_6):
538 + packed_modules_mapping = {
539 + "qkv_proj": [
540 + "q_proj",
541 + "k_proj",
542 + "v_proj",
543 + ],
544 + "gate_up_proj": [
545 + "gate_proj",
546 + "up_proj",
547 + ],
548 + }
549 +
550 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
551 + super().__init__(vllm_config=vllm_config, prefix=prefix)
552 + self.apm = self.init_audio_module(vllm_config=vllm_config,
553 + prefix=maybe_prefix(prefix, "apm"))
554 +
555 + def init_audio_module(self, *, vllm_config: VllmConfig, prefix: str = ""):
556 + # Do not use parameters temporarily
557 + audio_config = self.config.audio_config
558 + model = MiniCPMWhisperEncoder(audio_config)
559 + audio_output_dim = int(audio_config.encoder_ffn_dim // 4)
560 + self.audio_avg_pooler = \
561 + nn.AvgPool1d(self.config.audio_pool_step,
562 + stride=self.config.audio_pool_step)
563 + self.audio_projection_layer = \
564 + MultiModalProjector(in_dim=audio_output_dim,out_dim=self.embed_dim)
565 + self.audio_encoder_layer = -1
566 + return model
567 +
568 + def load_weights(self, weights: Iterable[Tuple[str,
569 + torch.Tensor]]) -> Set[str]:
570 + loader = AutoWeightsLoader(self, skip_prefixes=["tts"])
571 + return loader.load_weights(weights)
572 +
573 + def subsequent_chunk_mask(
574 + self,
575 + size: int,
576 + chunk_size: int,
577 + num_left_chunks: int = -1,
578 + device: torch.device = CPU_DEVICE,
579 + num_lookhead: int = 0,
580 + ) -> torch.Tensor:
581 + ret = torch.zeros(size, size, device=device, dtype=torch.bool)
582 + for i in range(size):
583 + if num_left_chunks < 0:
584 + start = 0
585 + else:
586 + start = max((i // chunk_size - num_left_chunks) * chunk_size,
587 + 0)
588 + ending = min((i // chunk_size + 1) * chunk_size + num_lookhead,
589 + size)
590 + ret[i, start:ending] = True
591 + return ret
592 +
593 + def _get_feat_extract_output_lengths(self,
594 + input_lengths: torch.LongTensor):
595 + input_lengths_after_cnn = (input_lengths - 1) // 2 + 1
596 + input_lengths_after_pooling = (
597 + input_lengths_after_cnn -
598 + self.config.audio_pool_step) // self.config.audio_pool_step + 1
599 + input_lengths_after_pooling = input_lengths_after_pooling.to(
600 + dtype=torch.int32)
601 +
602 + return input_lengths_after_cnn, input_lengths_after_pooling
603 +
604 + # Copied from HF repo of MiniCPM-o-2_6,
605 + # designed for batched inputs and outputs
606 + def get_audio_hidden_states(self, data: MiniCPMOAudioInputs,
607 + chunk_length: int) -> torch.Tensor:
608 + wavforms = data.get(
609 + "data",
610 + []) # (bs, 80, frames) or [], multi audios need filled in advance
611 + audio_feature_lens_raw = [data.get("audio_feature_lens",
612 + [])] # list, [[x1, x2], [y1], [z1]]
613 +
614 + # exist audio
615 + if len(wavforms) > 0:
616 + audio_feature_lens = torch.hstack(audio_feature_lens_raw)
617 + batch_size, _, max_mel_seq_len = wavforms.shape
618 + max_seq_len = (max_mel_seq_len - 1) // 2 + 1
619 +
620 + # Create a sequence tensor of shape (batch_size, max_seq_len)
621 + seq_range = (torch.arange(
622 + 0,
623 + max_seq_len,
624 + dtype=audio_feature_lens.dtype,
625 + device=audio_feature_lens.device).unsqueeze(0).expand(
626 + batch_size, max_seq_len))
627 + lengths_expand = audio_feature_lens.unsqueeze(1).expand(
628 + batch_size, max_seq_len)
629 + # Create mask
630 + padding_mask = seq_range >= lengths_expand # 1 for padded values
631 +
632 + audio_attention_mask_ = padding_mask.view(
633 + batch_size, 1, 1, max_seq_len).expand(batch_size, 1,
634 + max_seq_len, max_seq_len)
635 + audio_attention_mask = audio_attention_mask_.to(
636 + dtype=self.apm.conv1.weight.dtype,
637 + device=self.apm.conv1.weight.device)
638 +
639 + if chunk_length > 0:
640 + chunk_num_frame = int(chunk_length * 50)
641 + chunk_mask = self.subsequent_chunk_mask(
642 + size=max_seq_len,
643 + chunk_size=chunk_num_frame,
644 + num_left_chunks=-1,
645 + device=audio_attention_mask_.device,
646 + )
647 + audio_attention_mask_ = torch.logical_or(
648 + audio_attention_mask_, torch.logical_not(chunk_mask))
649 +
650 + audio_attention_mask[audio_attention_mask_] = float("-inf")
651 + audio_states = self.apm(
652 + wavforms, attention_mask=audio_attention_mask).hidden_states[
653 + self.audio_encoder_layer]
654 + audio_embeds = self.audio_projection_layer(audio_states)
655 +
656 + audio_embeds = audio_embeds.transpose(1, 2)
657 + audio_embeds = self.audio_avg_pooler(audio_embeds)
658 + audio_embeds = audio_embeds.transpose(1, 2)
659 +
660 + _, feature_lens_after_pooling = \
661 + self._get_feat_extract_output_lengths(audio_feature_lens)
662 +
663 + num_audio_tokens = feature_lens_after_pooling
664 +
665 + final_audio_embeds = []
666 + idx = 0
667 + for i in range(len(audio_feature_lens_raw)):
668 + target_audio_embeds = []
669 + for _ in range(len(audio_feature_lens_raw[i])):
670 + target_audio_embeds.append(
671 + audio_embeds[idx, :num_audio_tokens[idx], :])
672 + idx += 1
673 + final_audio_embeds.append(target_audio_embeds)
674 + return final_audio_embeds
675 + else:
676 + return []
677 +
678 + def get_embedding_with_audios(self, vlm_embedding: torch.Tensor,
679 + audio_inputs: Optional[MiniCPMOAudioInputs],
680 + chunk_length: int) -> torch.Tensor:
681 + device, dtype = vlm_embedding.device, vlm_embedding.dtype
682 + if audio_inputs["type"] == "audio_embeds":
683 + audio_embeddings = audio_inputs["data"]
684 + audio_embeddings = [
685 + audio_embeddings[i].to(device=device, dtype=dtype)
686 + for i in range(len(audio_embeddings))
687 + ]
688 + else:
689 + audio_embeddings = self.get_audio_hidden_states(
690 + audio_inputs, chunk_length)[0]
691 + if audio_embeddings is None or len(audio_embeddings) == 0:
692 + return vlm_embedding
693 + audio_bounds = audio_inputs["audio_bounds"]
694 + if self.config.chunk_input:
695 + audio_embs = torch.cat(audio_embeddings, dim=0).to(device=device,
696 + dtype=dtype)
697 + audio_start_pos = 0
698 + for bound in audio_bounds:
699 + audio_len = bound[1] - bound[0]
700 + vlm_embedding[bound[0]:bound[1]] = audio_embs[
701 + audio_start_pos:audio_start_pos + audio_len, :]
702 + audio_start_pos += audio_len
703 + else:
704 + for embs, bound in zip(audio_embeddings, audio_bounds):
705 + audio_indices = torch.arange(bound[0],
706 + bound[1],
707 + dtype=torch.long).to(device)
708 +
709 + if embs.shape[0] != len(audio_indices):
710 + raise ValueError(
711 + "Shape mismatch: Trying to assign embeddings "
712 + f"of shape {embs.shape} "
713 + f"to input indices of length {len(audio_indices)}")
714 + vlm_embedding[audio_indices] = embs.to(dtype)
715 + return vlm_embedding
716 +
717 + def _get_audio_bounds(self, input_ids: torch.Tensor,
718 + audio_start_id: torch.Tensor,
719 + audio_end_id: torch.Tensor) -> torch.Tensor:
720 + audio_start_tokens, = torch.where(input_ids == audio_start_id[0])
721 + audio_start_tokens += 1
722 + audio_end_tokens, = torch.where(input_ids == audio_end_id[0])
723 + valid_audio_nums = max(len(audio_start_tokens), len(audio_end_tokens))
724 + return torch.hstack([
725 + audio_start_tokens[:valid_audio_nums].unsqueeze(-1),
726 + audio_end_tokens[:valid_audio_nums].unsqueeze(-1)
727 + ])
728 +
729 + def _parse_and_validate_audio_inputs(
730 + self, input_ids: torch.Tensor,
731 + **kwargs: object) -> Tuple[MiniCPMOAudioInputs]:
732 + audio_features = kwargs.pop("audio_features", [])
733 + audio_feature_lens = kwargs.pop("audio_feature_lens", [])
734 + audio_embeds = kwargs.pop("audio_embeds", None)
735 + audio_start_id = kwargs.pop("audio_start_id", None)
736 + audio_end_id = kwargs.pop("audio_end_id", None)
737 + if audio_embeds is not None:
738 + audio_embeds = [
739 + audio_embeds[i][j] for i in range(len(audio_embeds))
740 + for j in range(len(audio_embeds[i]))
741 + ]
742 + return MiniCPMOAudioEmbeddingInputs(
743 + audio_bounds=self._get_audio_bounds(input_ids, audio_start_id,
744 + audio_end_id),
745 + data=audio_embeds,
746 + type="audio_embeds")
747 + if len(audio_features) > 0:
748 + audio_features_all = [
749 + i.permute(1, 0) for audio_feature in audio_features
750 + for i in audio_feature
751 + ]
752 + audio_features = torch.nn.utils.rnn.pad_sequence(
753 + audio_features_all, batch_first=True,
754 + padding_value=0.0).permute(0, 2, 1)
755 + audio_feature_lens = torch.cat(
756 + [item for item in audio_feature_lens])
757 +
758 + return MiniCPMOAudioFeatureInputs(
759 + audio_bounds=self._get_audio_bounds(input_ids, audio_start_id,
760 + audio_end_id),
761 + data=audio_features,
762 + audio_feature_lens=audio_feature_lens,
763 + type="audio_features")
764 + return None
765 +
766 + def _parse_and_validate_inputs(self, input_ids: torch.Tensor,
767 + **kwargs: object):
768 + image_inputs = self._parse_and_validate_image_inputs(
769 + input_ids, **kwargs)
770 + if not any("audio" in key for key in kwargs):
771 + return image_inputs, None
772 + audio_inputs = self._parse_and_validate_audio_inputs(
773 + input_ids, **kwargs)
774 + return image_inputs, audio_inputs
775 +
776 + def forward(
777 + self,
778 + input_ids: torch.Tensor,
779 + positions: torch.Tensor,
780 + kv_caches: List[torch.Tensor],
781 + attn_metadata: AttentionMetadata,
782 + intermediate_tensors: Optional[IntermediateTensors] = None,
783 + **kwargs: Any,
784 + ) -> torch.Tensor:
785 + if intermediate_tensors is not None:
786 + vlm_embeddings = None
787 + else:
788 + image_inputs, audio_inputs = \
789 + self._parse_and_validate_inputs(input_ids, **kwargs)
790 + vlm_embeddings, _ = self.get_embedding_with_vision(
791 + input_ids, image_inputs)
792 +
793 + if audio_inputs is not None:
794 + vlm_embeddings = self.get_embedding_with_audios(
795 + vlm_embeddings, audio_inputs,
796 + self.config.audio_chunk_length)
797 +
798 + # always pass the input via `inputs_embeds`
799 + # to make sure the computation graph is consistent
800 + # for `torch.compile` integration
801 + input_ids = None
802 +
803 + output = self.llm.model(
804 + input_ids=input_ids,
805 + positions=positions,
806 + kv_caches=kv_caches,
807 + attn_metadata=attn_metadata,
808 + intermediate_tensors=intermediate_tensors,
809 + inputs_embeds=vlm_embeddings,
810 + )
811 + return output
vllm/model_executor/models/minicpmv.py
+668 -175
@@ -22,21 +22,21 @@
22 """Inference-only MiniCPM-V model compatible with HuggingFace weights."""
23 import math
24 import re
25 +from collections import Counter
26 from functools import cached_property, partial
26 -from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
27 - Set, Tuple, TypedDict, Union)
27 +from itertools import accumulate
28 +from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
29 + Optional, Set, Tuple, TypedDict, Union)
30
31 +import numpy as np
32 import torch
33 import torch.types
34 from PIL import Image
35 from torch import nn
33 -from transformers import PretrainedConfig
34 -from typing_extensions import NotRequired
36 +from transformers import BatchFeature, PretrainedConfig
37
38 from vllm.attention import AttentionMetadata
39 from vllm.config import VllmConfig
38 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
39 - InputContext, token_inputs)
40 from vllm.model_executor.layers.quantization import QuantizationConfig
41 from vllm.model_executor.layers.resampler import (BaseResampler, Resampler2,
42 get_2d_sincos_pos_embed)
@@ -48,33 +48,30 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
48 from vllm.model_executor.models.qwen2 import Qwen2ForCausalLM
49 from vllm.model_executor.sampling_metadata import SamplingMetadata
50 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
51 -from vllm.multimodal.image import cached_get_image_processor
52 -from vllm.multimodal.utils import cached_get_tokenizer
53 -from vllm.sequence import IntermediateTensors, SequenceData
51 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
52 + MultiModalInputs, PlaceholderRange)
53 +from vllm.multimodal.parse import (ImageItem, ImageSize, ModalityData,
54 + ModalityDataItems, MultiModalDataItems,
55 + MultiModalDataParser, VideoItem)
56 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
57 + BaseProcessingInfo, PromptReplacement)
58 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
59 +from vllm.sequence import IntermediateTensors
60
61 from .idefics2_vision_model import Idefics2VisionTransformer
62 from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
63 from .utils import AutoWeightsLoader, maybe_prefix
64
59 -RawImageType = Union[Image.Image, torch.Tensor]
60 -
61 -
62 -class MiniCPMVRawImageInput(TypedDict):
63 - """Input mapper input with auxiliary data for computing image bounds."""
64 - image: RawImageType
65 +CPU_DEVICE = torch.device("cpu")
66
66 - # Image bounds token ids in 0-dim scaler tensor.
67 - im_start_id: torch.Tensor
68 - im_end_id: torch.Tensor
69 - slice_start_id: NotRequired[torch.Tensor]
70 - slice_end_id: NotRequired[torch.Tensor]
67 +RawImageType = Union[Image.Image, torch.Tensor]
68
69
70 class MiniCPMVImagePixelInputs(TypedDict):
71 type: Literal["pixel_values"]
72 data: List[torch.Tensor]
73 """
77 - Shape: `(batch_size * num_images, num_channels, height, width)`
74 + Shape: `(batch_size * num_images * num_slices, num_channels, height, width)`
75
76 Note that the image size may vary, so we pass it as a list
77 instead of a batched tensor.
@@ -82,14 +79,14 @@ class MiniCPMVImagePixelInputs(TypedDict):
79
80 image_bounds: torch.Tensor
81 """
85 - Shape: `(batch_size * num_images, 2)`
82 + Shape: `(batch_size * num_images * num_slices, 2)`
83
84 This should be in `(start, stop)` format.
85 """
86
87 tgt_sizes: torch.Tensor
88 """
92 - Shape: `(batch_size * num_images, 2)`
89 + Shape: `(batch_size * num_images * num_slices, 2)`
90
91 This should be in `(height, width)` format.
92 """
@@ -99,7 +96,8 @@ class MiniCPMVImageEmbeddingInputs(TypedDict):
96 type: Literal["image_embeds"]
97 data: torch.Tensor
98 """
102 - Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
99 + Shape: `(batch_size * num_images * num_slices,
100 + image_feature_size, hidden_size)`
101
102 `hidden_size` must match the hidden size of language model backbone.
103 instead of a batched tensor.
@@ -107,7 +105,7 @@ class MiniCPMVImageEmbeddingInputs(TypedDict):
105
106 image_bounds: torch.Tensor
107 """
110 - Shape: `(batch_size * num_images, 2)`
108 + Shape: `(batch_size * num_images * num_slices, 2)`
109
110 This should be in `(start, stop)` format.
111 """
@@ -116,6 +114,93 @@ class MiniCPMVImageEmbeddingInputs(TypedDict):
114 MiniCPMVImageInputs = Union[MiniCPMVImagePixelInputs,
115 MiniCPMVImageEmbeddingInputs]
116
117 +
118 +class MiniCPMVEmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
119 + dict[str, torch.Tensor]]):
120 +
121 + def __init__(self, data: Dict, modality: str) -> None:
122 + super().__init__(data, modality)
123 +
124 + def get_processor_data(self) -> Mapping[str, object]:
125 + return self.data
126 +
127 + def get_passthrough_data(self) -> Mapping[str, object]:
128 + return {}
129 +
130 + def get_count(self) -> int:
131 + return len(self.data[f"{self.modality}_embeds"])
132 +
133 + def get(self, index: int) -> Dict[str, torch.Tensor]:
134 + out = {}
135 + for k, v in self.data.items():
136 + out[k] = v[index]
137 + return out
138 +
139 +
140 +class MiniCPMVImageEmbeddingItems(MiniCPMVEmbeddingItems):
141 +
142 + def __init__(self, data: Dict) -> None:
143 + super().__init__(data, "image")
144 + image_embeds = self.data.get("image_embeds", None)
145 + image_sizes = self.data.get("image_sizes", None)
146 + if image_embeds is None:
147 + raise ValueError("In correct type of image_embeds",
148 + "Got type: None")
149 + if not isinstance(image_embeds[0], torch.Tensor):
150 + raise ValueError("In correct type of image_embeds",
151 + f"Got type: {type(image_embeds[0])}")
152 + if image_sizes is None:
153 + raise ValueError(
154 + "In correct type of image_sizes", "Got type: None."
155 + "If you're using `image_size_list`, "
156 + "please rename it to `image_sizes`")
157 + if len(image_embeds[0].shape) == 2:
158 + image_embeds = [image_embeds]
159 + image_sizes = [image_sizes]
160 + self.data["image_embeds"] = image_embeds
161 + self.data["image_sizes"] = image_sizes
162 +
163 + def get_image_size(self, index: int) -> ImageSize:
164 + image_size = self.data["image_sizes"][index]
165 + return ImageSize(width=image_size[0], height=image_size[1])
166 +
167 +
168 +class MiniCPMVVideoEmbeddingItems(MiniCPMVEmbeddingItems):
169 +
170 + def __init__(self, data: Dict) -> None:
171 + super().__init__(data, "video")
172 + video_embeds = self.data.get("video_embeds", None)
173 + image_sizes = self.data.get("image_sizes", None)
174 + num_frames = self.data.get("num_frames", None)
175 + if video_embeds is None:
176 + raise ValueError("In correct type of video_embeds",
177 + "Got type: None")
178 + if not isinstance(video_embeds[0], torch.Tensor):
179 + raise ValueError("In correct type of video_embeds",
180 + f"Got type: {type(video_embeds[0])}")
181 + if image_sizes is None:
182 + raise ValueError(
183 + "In correct type of image_sizes", "Got type: None."
184 + "If you're using `image_size_list`, "
185 + "please rename it to `image_sizes`")
186 + if num_frames is None:
187 + raise ValueError("In correct type of numframes", "Got type: None")
188 + if len(video_embeds[0].shape) == 2:
189 + video_embeds = [video_embeds]
190 + image_sizes = [image_sizes]
191 + num_frames = [num_frames]
192 + self.data["video_embeds"] = video_embeds
193 + self.data["image_sizes"] = image_sizes
194 + self.data["num_frames"] = num_frames
195 +
196 + def get_frame_size(self, index: int) -> ImageSize:
197 + frame_size = self.data["image_sizes"][index]
198 + return ImageSize(width=frame_size[0], height=frame_size[1])
199 +
200 + def get_num_frames(self, index: int) -> int:
201 + return self.data["num_frames"][index]
202 +
203 +
204 DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
205
206
@@ -212,25 +297,6 @@ class Resampler2_5(BaseResampler):
297 return x
298
299
215 -def _build_image_input(ctx: InputContext,
216 - image: RawImageType) -> MiniCPMVRawImageInput:
217 - tokenizer = cached_get_tokenizer(
218 - ctx.model_config.tokenizer,
219 - trust_remote_code=ctx.model_config.trust_remote_code)
220 - if hasattr(tokenizer, "slice_start_id"):
221 - return MiniCPMVRawImageInput(
222 - image=image,
223 - im_start_id=torch.tensor(tokenizer.im_start_id),
224 - im_end_id=torch.tensor(tokenizer.im_end_id),
225 - slice_start_id=torch.tensor(tokenizer.slice_start_id),
226 - slice_end_id=torch.tensor(tokenizer.slice_end_id))
227 - else:
228 - return MiniCPMVRawImageInput(
229 - image=image,
230 - im_start_id=torch.tensor(tokenizer.im_start_id),
231 - im_end_id=torch.tensor(tokenizer.im_end_id))
232 -
233 -
300 def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]:
301 version_float = getattr(config, "version", None)
302
@@ -240,129 +306,512 @@ def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]:
306 if config.hidden_size == 2304 and config.query_num == 64:
307 return (2, 0)
308 return (2, 5)
243 -
309 version_str = str(version_float)
310 return tuple(int(x) for x in version_str.split("."))
311
312
248 -def get_max_minicpmv_image_tokens(ctx: InputContext):
249 - hf_config = ctx.get_hf_config()
250 - return getattr(hf_config, "query_num", 64)
313 +class MiniCPMVMultiModalDataParser(MultiModalDataParser):
314 +
315 + def _parse_image_data(
316 + self,
317 + data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
318 + ) -> ModalityDataItems[Any, Any]:
319 + if isinstance(data, dict):
320 + return MiniCPMVImageEmbeddingItems(data)
321 + return super()._parse_image_data(data)
322 +
323 + def _parse_video_data(
324 + self,
325 + data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],
326 + ) -> ModalityDataItems[Any, Any]:
327 + if isinstance(data, dict):
328 + return MiniCPMVVideoEmbeddingItems(data)
329 + return super()._parse_video_data(data)
330 +
331 +
332 +class MiniCPMVProcessingInfo(BaseProcessingInfo):
333 + image_pattern = "(<image>./</image>)"
334 + video_pattern = "(<video>./</video>)"
335 +
336 + def get_hf_config(self):
337 + return self.ctx.get_hf_config()
338 +
339 + def get_hf_processor(
340 + self,
341 + **kwargs: object,
342 + ):
343 + hf_processor = self.ctx.get_hf_processor()
344 + return hf_processor
345 +
346 + def get_image_processor(self):
347 + hf_processor = self.get_hf_processor()
348 + image_processor = hf_processor.image_processor # type: ignore
349 + return image_processor
350 +
351 + def get_model_version(self):
352 + return get_version_by_config(self.get_hf_config())
353 +
354 + def get_supported_mm_modalities(self) -> List[str]:
355 + if self.get_model_version() == (2, 6):
356 + return ["image", "video"]
357 + else:
358 + return ["image"]
359 +
360 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
361 + if self.get_model_version() == (2, 6):
362 + return {"image": None, "video": None}
363 + else:
364 + return {"image": None}
365 +
366 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
367 + mm_max_tokens = {"image": self.get_max_image_tokens()}
368 + if self.get_model_version() == (2, 6):
369 + mm_max_tokens["video"] = self.get_max_video_tokens(seq_len)
370 + return mm_max_tokens
371 +
372 + def get_max_video_frame_tokens(self) -> int:
373 + frame_size = self.get_video_frame_size_with_most_features()
374 + return self.get_num_image_tokens(frame_size,
375 + self.get_video_max_slice_num())
376 +
377 + def get_max_video_tokens(self, seq_len: int) -> int:
378 + return self.get_max_video_frame_tokens(
379 + ) * self.get_num_frames_with_most_features(seq_len)
380 +
381 + def get_max_audio_tokens(self) -> int:
382 + return self.get_max_audio_tokens_per_chunk(
383 + ) * self.get_max_audio_chunks_with_most_features()
384 +
385 + def get_slice_query_num(self) -> int:
386 + hf_config = self.get_hf_config()
387 + query_num = getattr(hf_config, "query_num", 64)
388 + return query_num
389 +
390 + def get_max_slice_num(self) -> int:
391 + hf_config = self.get_hf_config()
392 + max_slice_num = getattr(hf_config, "max_slice_num", 9)
393 + return max_slice_num
394 +
395 + def get_sliced_grid(self, image_size: ImageSize,
396 + max_slice_num: int) -> Tuple[int, int]:
397 + if self.get_model_version() == (2, 6):
398 + slice_grid = self.get_image_processor().get_sliced_grid(
399 + image_size, max_slice_num)
400 + else:
401 + slice_grid = self.get_image_processor().get_sliced_grid(image_size)
402 + return slice_grid
403 +
404 + def get_num_image_tokens(self, image_size: ImageSize,
405 + max_slice_num: int) -> int:
406 + slice_grid = self.get_sliced_grid(image_size, max_slice_num)
407 + num_tokens = self.get_slice_query_num(
408 + ) + 2 # <image>(<unk> * query_num)</image>
409 + if slice_grid is not None:
410 + if self.get_model_version() == (2, 6):
411 + num_additional_tokens = 0
412 + else:
413 + # <slice><image>(<unk> * query_num)</image></slice>
414 + num_additional_tokens = 2
415 + num_tokens += ((self.get_slice_query_num() + 2) \
416 + * slice_grid[0] * slice_grid[1]) \
417 + + slice_grid[1] - 1 + num_additional_tokens
418 + return num_tokens
419
420 + def get_image_slice_nums(self, image_size: torch.Tensor,
421 + max_slice_nums: int) -> int:
422 + grid = self.get_sliced_grid(image_size, max_slice_nums)
423 + return 1 if grid is None else grid[0] * grid[1] + 1
424
253 -def dummy_seq_data_for_minicpmv(seq_len: int, num_images: int):
254 - return SequenceData.from_prompt_token_counts((0, seq_len))
425 + def get_max_image_tokens(self) -> int:
426 + image_size = self.get_image_size_with_most_features()
427 + return self.get_num_image_tokens(image_size, self.get_max_slice_num())
428
429 + def get_image_size_with_most_features(self) -> ImageSize:
430 + # Result in the max possible feature size (h:w = 9:1)
431 + return self.get_default_image_sizes(self.get_max_slice_num())
432
257 -def dummy_image_for_minicpmv(ctx: InputContext, hf_config: PretrainedConfig,
258 - num_images: int):
259 - width = height = hf_config.image_size
260 - image = _build_image_input(ctx,
261 - image=Image.new("RGB", (width, height),
262 - color=0))
263 - return {"image": [image] if num_images == 1 else [image] * num_images}
433 + def get_video_max_slice_num(self) -> int:
434 + return 1
435
436 + def get_video_frame_size_with_most_features(self) -> ImageSize:
437 + return self.get_default_image_sizes(self.get_video_max_slice_num())
438
266 -def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int,
267 - mm_counts: Mapping[str, int]):
268 - hf_config = ctx.get_hf_config()
269 - num_images = mm_counts["image"]
439 + def get_max_video_frames(self, max_tokens: int) -> int:
440 + num_frame_tokens = self.get_max_video_frame_tokens()
441 + num_frames = max_tokens // num_frame_tokens
442 + return num_frames
443
271 - seq_data = dummy_seq_data_for_minicpmv(seq_len, num_images)
272 - mm_data = dummy_image_for_minicpmv(ctx, hf_config, num_images)
444 + def get_num_frames_with_most_features(self, seq_len: int) -> int:
445 + mm_config = self.ctx.get_mm_config()
446 + max_images = mm_config.limit_per_prompt.get("image", 1)
447 + max_videos = mm_config.limit_per_prompt.get("video", 1)
448
274 - return DummyData(seq_data, mm_data)
449 + # count <image_idx></image_idx> tokens
450 + # which are not in get_max_image_tokens
451 + max_image_tokens = self.get_max_image_tokens(
452 + ) * max_images + 4 * max_images
453 + max_total_frames = self.get_max_video_frames(seq_len -
454 + max_image_tokens)
455
456 + num_frames = max(max_total_frames // max(max_videos, 1), 1)
457
277 -def input_processor_for_minicpmv(ctx: InputContext, inputs: DecoderOnlyInputs):
278 - multi_modal_data = inputs.get("multi_modal_data")
279 - if multi_modal_data is None or "image" not in multi_modal_data:
280 - return inputs
281 - model_config = ctx.model_config
282 - version = get_version_by_config(model_config.hf_config)
283 - tokenizer = cached_get_tokenizer(
284 - model_config.tokenizer,
285 - trust_remote_code=model_config.trust_remote_code)
286 - image_processor = cached_get_image_processor(model_config.tokenizer)
458 + return num_frames
459
288 - def get_placeholder(image_size: Tuple[int, int], num_image: int):
460 + def get_default_image_sizes(self, num_slices: int) -> ImageSize:
461 + image_size = getattr(self.get_hf_config(), "image_size", 448)
462 + return ImageSize(width=image_size, height=image_size * num_slices)
463 +
464 +
465 +class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[MiniCPMVProcessingInfo]
466 + ):
467 +
468 + def get_dummy_processor_inputs(
469 + self,
470 + seq_len: int,
471 + mm_counts: Mapping[str, int],
472 + ) -> ProcessorInputs:
473 + num_images = mm_counts.get("image", 0)
474 + num_videos = mm_counts.get("video", 0)
475 +
476 + image_width, image_height = \
477 + self.info.get_image_size_with_most_features()
478 + video_width, video_height = \
479 + self.info.get_video_frame_size_with_most_features()
480 + num_video_frames = \
481 + self.info.get_num_frames_with_most_features(seq_len)
482 +
483 + mm_data = {
484 + "image":
485 + self._get_dummy_images(width=image_width,
486 + height=image_height,
487 + num_images=num_images),
488 + "video": [
489 + self._get_dummy_images(width=video_width,
490 + height=video_height,
491 + num_images=num_video_frames)
492 + ] * num_videos,
493 + }
494 +
495 + image_prompt_texts = self.info.image_pattern * num_images
496 + video_prompt_texts = self.info.video_pattern * num_videos
497 +
498 + return ProcessorInputs(prompt_text=image_prompt_texts +
499 + video_prompt_texts,
500 + mm_data=mm_data)
501 +
502 +
503 +class MiniCPMVMultiModalProcessor(
504 + BaseMultiModalProcessor[MiniCPMVProcessingInfo]):
505 +
506 + def _get_data_parser(self) -> MultiModalDataParser:
507 + return MiniCPMVMultiModalDataParser()
508 +
509 + def get_slice_image_placeholder(self, image_size: ImageSize,
510 + **kwargs) -> str:
511 + image_processor = self.info.get_image_processor()
512 + version = self.info.get_model_version()
513 if version == (2, 0) or version == (2, 5):
514 return image_processor.get_slice_image_placeholder(image_size)
515 return image_processor.get_slice_image_placeholder(
292 - image_size, num_image)
293 -
294 - prompt = inputs.get("prompt")
295 - token_ids = inputs.get("prompt_token_ids")
296 - if prompt is None:
297 - prompt = tokenizer.decode(token_ids)
298 -
299 - pattern = "(<image>./</image>)"
300 - images = multi_modal_data["image"]
301 - image_tags = re.findall(pattern, prompt)
302 - if len(image_tags) == 0:
303 - new_token_ids = token_ids
304 - new_prompt = prompt
305 - else:
306 - if isinstance(images, dict):
307 - image_size_list = images.get("image_size_list")
308 - images = [images.get("image_embeds")]
516 + image_size, **kwargs)
517 +
518 + def get_image_prompt_texts(self,
519 + image_size: ImageSize,
520 + image_idx: int = 0) -> str:
521 + prompt_texts = self.get_slice_image_placeholder(image_size,
522 + image_idx=image_idx)
523 + return prompt_texts
524 +
525 + def get_video_prompt_texts(self, image_size: ImageSize,
526 + num_frames: int) -> str:
527 + prompt_texts = "".join(
528 + self.get_slice_image_placeholder(
529 + image_size=image_size,
530 + image_idx=0,
531 + max_slice_nums=self.info.get_video_max_slice_num(),
532 + use_image_id=False) for image_idx in range(num_frames))
533 + return prompt_texts
534 +
535 + def get_special_tokens(self) -> Dict[str, torch.Tensor]:
536 + tokenizer = self.info.get_tokenizer()
537 + special_tokens = {
538 + "im_start_id": torch.tensor(tokenizer.im_start_id),
539 + "im_end_id": torch.tensor(tokenizer.im_end_id)
540 + }
541 + if hasattr(tokenizer, "slice_start_id"):
542 + special_tokens["slice_start_id"] = torch.tensor(
543 + tokenizer.slice_start_id)
544 + special_tokens["slice_end_id"] = torch.tensor(
545 + tokenizer.slice_end_id)
546 + return special_tokens
547 +
548 + @staticmethod
549 + def repack_processor_outputs(outputs: Any) -> BatchFeature:
550 + valid_keys = ["pixel_values", "image_sizes", "tgt_sizes"]
551 + outputs = {key: outputs[key][0] for key in valid_keys}
552 + return outputs
553 +
554 + def process_images(self, mm_data: Mapping[str, object],
555 + mm_kwargs: Mapping[str, object]) -> Dict[str, object]:
556 + images = mm_data.pop("images", [])
557 + image_embeds = mm_data.pop("image_embeds", [])
558 + if isinstance(images, Image.Image):
559 + images = [images]
560 + if isinstance(images, (list, torch.Tensor)) and len(images) > 0:
561 + image_outputs = super()._call_hf_processor(
562 + prompt=self.info.image_pattern * len(images),
563 + mm_data={"images": images},
564 + mm_kwargs=mm_kwargs)
565 + image_outputs = MiniCPMVMultiModalProcessor.\
566 + repack_processor_outputs(image_outputs)
567 + elif len(image_embeds) > 0:
568 + image_sizes = mm_data.pop("image_sizes", None)
569 + image_outputs = {
570 + "image_embeds": torch.cat(image_embeds),
571 + "image_sizes": image_sizes
572 + }
573 else:
310 - if isinstance(images, Image.Image):
311 - images = [images]
312 - image_size_list = [image.size for image in images]
313 -
314 - text_chunks = prompt.split(pattern)
315 - new_prompt_chunks: List[str] = []
316 - for i in range(len(image_size_list)):
317 - new_prompt_chunks += [
318 - text_chunks[i],
319 - get_placeholder(image_size_list[i], i)
320 - ]
321 - new_prompt_chunks.append(text_chunks[-1])
322 - new_prompt = "".join(new_prompt_chunks)
323 - new_token_ids = tokenizer.encode(new_prompt)
324 -
325 - multi_modal_data["image"] = [
326 - _build_image_input(ctx, image) for image in images
327 - ]
574 + image_outputs = {}
575 + return image_outputs
576 +
577 + def process_videos(self, mm_data: Mapping[str, object],
578 + mm_kwargs: Mapping[str, object]) -> Dict[str, object]:
579 + videos = mm_data.pop("videos", [])
580 + video_embeds = mm_data.pop("video_embeds", [])
581 + if len(videos) > 0 and isinstance(videos[0], Image.Image):
582 + videos = [videos]
583 + if isinstance(videos, list) and len(videos) > 0:
584 + video_outputs = {
585 + "video_pixel_values": [],
586 + "video_image_sizes": [],
587 + "video_tgt_sizes": [],
588 + "num_frames": []
589 + }
590 + for video in videos:
591 + parsed_video = []
592 + for frame in video:
593 + if isinstance(frame, np.ndarray):
594 + parsed_video.append(Image.fromarray(frame))
595 + else:
596 + parsed_video.append(frame)
597 + video = parsed_video
598 + single_video_outputs = super()._call_hf_processor(
599 + prompt=self.info.image_pattern * len(video),
600 + mm_data={"images": video},
601 + mm_kwargs={
602 + **mm_kwargs, "max_slice_nums":
603 + self.info.get_video_max_slice_num()
604 + })
605 + video_outputs["num_frames"].append(len(video))
606 + for key in single_video_outputs:
607 + if "video_" + key in video_outputs:
608 + if key == "image_sizes":
609 + video_outputs["video_" + key].append(
610 + single_video_outputs[key][0][0])
611 + else:
612 + video_outputs["video_" +
613 + key] += single_video_outputs[key][0]
614 + elif len(video_embeds):
615 + image_sizes = mm_data.pop("image_sizes", None)
616 + num_frames = mm_data.pop("num_frames", None)
617 + video_outputs = {
618 + "video_embeds": torch.cat(video_embeds),
619 + "video_image_sizes": image_sizes,
620 + "num_frames": num_frames
621 + }
622 + else:
623 + video_outputs = {}
624 + return video_outputs
625
329 - return token_inputs(
330 - prompt_token_ids=new_token_ids,
331 - prompt=new_prompt,
332 - multi_modal_data=multi_modal_data,
333 - )
626 + def get_placeholder_match_pattern(self) -> str:
627 + return r"\(<(image|video)>./</\1>\)"
628
629 + def get_placeholder_split_pattern(self) -> str:
630 + return r"\(<(?:image|video)>./</(?:image|video)>\)"
631
336 -def input_mapper_for_minicpmv(ctx: InputContext, data: object):
337 - model_config = ctx.model_config
632 + def process_mm_inputs(self, mm_data, mm_kwargs) -> object:
633 + return {
634 + "image": self.process_images(mm_data, mm_kwargs),
635 + "video": self.process_videos(mm_data, mm_kwargs)
636 + }
637
339 - image_processor = cached_get_image_processor(
340 - model_config.model, trust_remote_code=model_config.trust_remote_code)
341 - if image_processor is None:
342 - raise RuntimeError("No HuggingFace processor is available "
343 - "to process the image object")
638 + def get_input_modalities(self, mm_data) -> List[str]:
639 + supported_mm_modalities = self.info.get_supported_mm_modalities()
640 + input_modalities = []
641 + for modality in supported_mm_modalities:
642 + if modality in mm_data and mm_data[modality] != {}:
643 + input_modalities.append(modality)
644 + return input_modalities
645 +
646 + def get_modality_num_counter(self, modality: str) -> str:
647 + if modality == "image":
648 + return "image_sizes"
649 + elif modality == "video":
650 + return "video_image_sizes"
651 +
652 + def get_num_slices_by_modality(self, inputs: Dict[str, object],
653 + modality: str, index: int) -> int:
654 + if modality == "image":
655 + return self.info.get_image_slice_nums(
656 + inputs[modality]["image_sizes"][index],
657 + self.info.get_max_slice_num())
658 + elif modality == "video":
659 + return self.info.get_image_slice_nums(
660 + inputs[modality]["video_image_sizes"][index],
661 + self.info.get_video_max_slice_num()
662 + ) * inputs[modality]["num_frames"][index]
663 + else:
664 + raise ValueError(f"UnExpected modality: {modality}")
665 +
666 + def check_mm_inputs(self, inputs: Dict[str, object],
667 + matches: List[str]) -> None:
668 + counts = Counter(matches)
669 + for modality, count in counts.items():
670 + if modality not in inputs or not inputs[modality]:
671 + raise ValueError(f"None input data of {modality}."
672 + "But prompt requires.")
673 + counter_key = self.get_modality_num_counter(modality)
674 + if len(inputs[modality][counter_key]) != count:
675 + raise ValueError(f"The prompt requires {count} "
676 + f"{modality} inputs while you pass "
677 + f"{len(inputs[modality][counter_key])}")
678 +
679 + def get_prompt_texts_by_modality(self, inputs: Dict[str, object],
680 + modality: str, index: int) -> str:
681 + if modality == "image":
682 + return self.get_image_prompt_texts(
683 + inputs["image"]["image_sizes"][index], index)
684 + elif modality == "video":
685 + return self.get_video_prompt_texts(
686 + inputs["video"]["video_image_sizes"][index],
687 + inputs["video"]["num_frames"][index])
688 + else:
689 + raise ValueError(f"UnExpected modality: {modality}")
690
345 - if not isinstance(data, list):
346 - raise ValueError(
347 - "Image input must be list of MiniCPMVImageInput, got (%s)", data)
691 + def call_base_hf_processor(
692 + self,
693 + prompt: str,
694 + mm_data: Mapping[str, object],
695 + mm_kwargs: Mapping[str, object],
696 + ) -> BatchFeature:
697 + return super()._call_hf_processor(prompt=prompt,
698 + mm_data=mm_data,
699 + mm_kwargs=mm_kwargs)
700 +
701 + def _call_hf_processor(
702 + self,
703 + prompt: str,
704 + mm_data: Mapping[str, object],
705 + mm_kwargs: Mapping[str, object],
706 + ) -> BatchFeature:
707 + # Do not support combination inputs of images and videos for now
708 + # Try to handle interleaved multimodal data
709 + tokenizer = self.info.get_tokenizer()
710 + inputs = self.process_mm_inputs(mm_data, mm_kwargs)
711 + mm_input_modalities = self.get_input_modalities(inputs)
712 + num_mm_slices = {modality: [] for modality in mm_input_modalities}
713 + for modality in mm_input_modalities:
714 + num_counter_key = self.get_modality_num_counter(modality)
715 + for index in range(len(inputs[modality][num_counter_key])):
716 + num_mm_slices[modality].append(
717 + self.get_num_slices_by_modality(inputs, modality, index))
718 + return {
719 + "input_ids": np.array([tokenizer.encode(prompt)]),
720 + **{
721 + key: value
722 + for modality in inputs
723 + for key, value in inputs[modality].items()
724 + },
725 + **{
726 + f"{modality}_num_slices": num_mm_slices[modality]
727 + for modality in mm_input_modalities
728 + }
729 + }
730
349 - if len(data) > 0 and isinstance(data[0]['image'], torch.Tensor):
350 - batch_data = {
351 - "image_embeds": data[0]['image'],
731 + def _get_prompt_replacements(
732 + self, mm_items: MultiModalDataItems,
733 + hf_processor_mm_kwargs: Mapping[str, Any],
734 + out_mm_kwargs: MultiModalKwargs) -> List[PromptReplacement]:
735 + placeholder = {
736 + "image": self.info.image_pattern,
737 + "video": self.info.video_pattern,
738 }
353 - else:
354 - batch_data = image_processor \
355 - .preprocess([img["image"] for img in data], return_tensors="pt") \
356 - .data
739
358 - if len(data) > 0:
359 - batch_data["im_start_id"] = data[0]["im_start_id"]
360 - batch_data["im_end_id"] = data[0]["im_end_id"]
361 - if "slice_start_id" in data[0]:
362 - batch_data["slice_start_id"] = data[0]["slice_start_id"]
363 - batch_data["slice_end_id"] = data[0]["slice_end_id"]
740 + def get_replacement_minicpmv(item_idx: int, modality: str):
741 + if modality == "image":
742 + return self.get_image_prompt_texts(
743 + mm_items["image"].get_image_size(item_idx), item_idx)
744 + else: # video
745 + return self.get_video_prompt_texts(
746 + mm_items["video"].get_frame_size(item_idx),
747 + mm_items["video"].get_num_frames(item_idx))
748 +
749 + return [
750 + PromptReplacement(modality=modality,
751 + target=placeholder[modality],
752 + replacement=partial(get_replacement_minicpmv,
753 + modality=modality))
754 + for modality in ("image", "video")
755 + ]
756
365 - return MultiModalKwargs(batch_data)
757 + def _get_mm_fields_config(
758 + self,
759 + hf_inputs,
760 + hf_processor_mm_kwargs: Mapping[str, object],
761 + ) -> Mapping[str, MultiModalFieldConfig]:
762 +
763 + def get_slices(num_slices: List[int]) -> List[int]:
764 + slice_indices = [0] + list(accumulate(num_slices))
765 + slices = [(slice_indices[i], slice_indices[i + 1])
766 + for i in range(len(num_slices))]
767 + return [slice(*slice_item) for slice_item in slices]
768 +
769 + image_slices = get_slices(
770 + hf_inputs.get("image_num_slices", torch.empty(0)))
771 + video_slices = get_slices(
772 + hf_inputs.get("video_num_slices", torch.empty(0)))
773 +
774 + return dict(
775 + pixel_values=MultiModalFieldConfig.flat("image", image_slices),
776 + image_sizes=MultiModalFieldConfig.batched("image"),
777 + tgt_sizes=MultiModalFieldConfig.flat("image", image_slices),
778 + image_num_slices=MultiModalFieldConfig.batched("image"),
779 + image_embeds=MultiModalFieldConfig.flat("image", image_slices),
780 + video_pixel_values=MultiModalFieldConfig.flat(
781 + "video", video_slices),
782 + video_image_sizes=MultiModalFieldConfig.batched("video"),
783 + video_tgt_sizes=MultiModalFieldConfig.flat("video", video_slices),
784 + video_embeds=MultiModalFieldConfig.flat("video", video_slices),
785 + video_num_slices=MultiModalFieldConfig.batched("video"))
786 +
787 + def apply(
788 + self,
789 + prompt: Union[str, List[int]],
790 + mm_data: MultiModalDataDict,
791 + hf_processor_mm_kwargs: Mapping[str, object],
792 + ) -> MultiModalInputs:
793 + supported_mm_modalities = self.info.get_supported_mm_modalities()
794 + if isinstance(prompt, list):
795 + prompt = self.info.get_tokenizer().decode(prompt)
796 + matches = re.findall(self.get_placeholder_match_pattern(), prompt)
797 + mm_orders = {
798 + f"{modality}_orders":
799 + torch.tensor(
800 + [index for index, m in enumerate(matches) if m == modality])
801 + for modality in supported_mm_modalities
802 + }
803 + result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
804 + # Exclude <image_id>x</image_id> from placeholders
805 + if "image" in result["mm_placeholders"] and \
806 + self.info.get_model_version() == (2, 6):
807 + result["mm_placeholders"]["image"] = [
808 + PlaceholderRange(offset=p["offset"] + 3 + idx // 10,
809 + length=p["length"] - 3 - idx // 10)
810 + for idx, p in enumerate(result["mm_placeholders"]["image"])
811 + ]
812 + result["mm_kwargs"].update(**mm_orders)
813 + result["mm_kwargs"].update(**self.get_special_tokens())
814 + return result
815
816
817 class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
@@ -409,7 +858,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
858
859 return get_sampler()
860
412 - def get_embedding(
861 + def get_embedding_with_vision(
862 self,
863 input_ids: torch.Tensor,
864 image_inputs: Optional[MiniCPMVImageInputs],
@@ -471,25 +920,46 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
920 image_end_tokens[:valid_image_nums].unsqueeze(-1),
921 ])
922
474 - def _parse_and_validate_inputs(
923 + def _parse_and_validate_image_inputs(
924 self,
925 input_ids: torch.Tensor,
926 **kwargs: object,
927 ) -> Optional[MiniCPMVImageInputs]:
479 - pixel_values = kwargs.pop("pixel_values", [])
480 - tgt_sizes = kwargs.pop("tgt_sizes", [])
928 + mm_data = {
929 + "image": {
930 + key: kwargs.pop(key, [])
931 + for key in ["pixel_values", "tgt_sizes", "image_num_slices"]
932 + },
933 + "video": {
934 + "pixel_values": kwargs.pop("video_pixel_values", []),
935 + "tgt_sizes": kwargs.pop("video_tgt_sizes", []),
936 + "video_num_slices": kwargs.pop("video_num_slices", [])
937 + }
938 + }
939 im_start_id = kwargs.pop("im_start_id", None)
940 im_end_id = kwargs.pop("im_end_id", None)
941 slice_start_id = kwargs.pop("slice_start_id", None)
942 slice_end_id = kwargs.pop("slice_end_id", None)
943 + mm_orders = {
944 + f"{modality}": kwargs.pop(f"{modality}_orders", None)
945 + for modality in ["image", "video", "audio"]
946 + }
947 + batch_size = max(len(mm_data["image"]["pixel_values"]),
948 + len(mm_data["video"]["pixel_values"]))
949 image_embeds = kwargs.pop("image_embeds", None)
486 -
950 + video_embeds = kwargs.pop("video_embeds", None)
951 + if image_embeds is not None and video_embeds is not None:
952 + raise ValueError(
953 + "Incorrect inputs for vision embeddings. "
954 + "Image embeds and video embeds can not exist simultaneously.")
955 + if video_embeds is not None:
956 + image_embeds = video_embeds
957 if image_embeds is not None:
958 if not isinstance(image_embeds, (torch.Tensor, list)):
959 raise ValueError(f"Incorrect type of image embeds. "
960 f"Got type: {type(image_embeds)}")
491 - if isinstance(image_embeds, list):
492 - image_embeds = torch.concat(image_embeds)
961 + image_embeds = torch.concat(
962 + [image_embeds[i] for i in range(len(image_embeds))])
963
964 return MiniCPMVImageEmbeddingInputs(
965 image_bounds=self._get_image_bounds(input_ids, im_start_id,
@@ -498,29 +968,47 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
968 data=image_embeds,
969 type="image_embeds",
970 )
501 -
502 - if not isinstance(pixel_values, (torch.Tensor, list)):
503 - raise ValueError("Incorrect type of pixel values. "
504 - f"Got type: {type(pixel_values)}")
505 -
506 - if not isinstance(tgt_sizes, (torch.Tensor, list)):
507 - raise ValueError("Incorrect type of target sizes. "
508 - f"Got type: {type(tgt_sizes)}")
509 -
510 - if len(pixel_values) != len(tgt_sizes):
511 - raise ValueError("Inconsistent batch lengths, found: "
512 - f"{len(pixel_values)} vs. {len(tgt_sizes)}")
971 + for modality, modality_mm_data in mm_data.items():
972 + if not isinstance(modality_mm_data["pixel_values"],
973 + (torch.Tensor, list)):
974 + raise ValueError(
975 + "Incorrect type of pixel values. "
976 + f"Got type: {type(modality_mm_data['pixel_values'])}")
977 +
978 + if not isinstance(modality_mm_data["tgt_sizes"],
979 + (torch.Tensor, list)):
980 + raise ValueError(
981 + "Incorrect type of target sizes. "
982 + f"Got type: {type(modality_mm_data['tgt_sizes'])}")
983 +
984 + if len(modality_mm_data["pixel_values"]) != len(
985 + modality_mm_data["tgt_sizes"]):
986 + raise ValueError(
987 + "Inconsistent batch lengths, found: "
988 + f"{len(modality_mm_data['pixel_values'])} vs. "
989 + f"{len(modality_mm_data['tgt_sizes'])}")
990
991 pixel_values_flat: List[torch.Tensor] = []
992 tgt_sizes_flat: List[torch.Tensor] = []
516 - for pixel_b, tgt_b in zip(pixel_values, tgt_sizes):
517 - if len(pixel_b) != len(tgt_b):
518 - raise ValueError("Inconsistent N lengths, found: "
519 - f"{len(pixel_b)} vs {len(tgt_b)}")
520 -
521 - for pixel_n, tgt_n in zip(pixel_b, tgt_b):
522 - pixel_values_flat += pixel_n
523 - tgt_sizes_flat += tgt_n
993 + for b in range(batch_size):
994 + mm_counts = {"image": 0, "video": 0} if self.version == (2, 6) \
995 + else {"image": 0}
996 + mm_slice_counts = {"image": 0, "video": 0} \
997 + if self.version == (2, 6) else {"image": 0}
998 + mm_orders_b = [(index, modality) for modality in mm_counts
999 + for index in mm_orders[modality][b]]
1000 + for _, modality in sorted(mm_orders_b, key=lambda x: x[0]):
1001 + pos = mm_counts[modality]
1002 + num_slices = mm_data[modality][f"{modality}_num_slices"][b][
1003 + pos]
1004 + slice_start_idx = mm_slice_counts[modality]
1005 + slice_end_idx = slice_start_idx + num_slices
1006 + pixel_values_flat += mm_data[modality]["pixel_values"][b][
1007 + slice_start_idx:slice_end_idx]
1008 + tgt_sizes_flat += mm_data[modality]["tgt_sizes"][b][
1009 + slice_start_idx:slice_end_idx]
1010 + mm_counts[modality] += 1
1011 + mm_slice_counts[modality] += num_slices
1012
1013 # NOTE: Input IDs does not contain image tokens during memory profiling,
1014 # so we allow it to be empty
@@ -544,6 +1032,10 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
1032 type="pixel_values",
1033 )
1034
1035 + def _parse_and_validate_inputs(self, input_ids: torch.Tensor,
1036 + **kwargs: object):
1037 + return self._parse_and_validate_image_inputs(input_ids, **kwargs)
1038 +
1039 def forward(
1040 self,
1041 input_ids: torch.Tensor,
@@ -556,9 +1048,10 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
1048 if intermediate_tensors is not None:
1049 vlm_embeddings = None
1050 else:
559 - image_inputs = self._parse_and_validate_inputs(input_ids, **kwargs)
560 -
561 - vlm_embeddings, _ = self.get_embedding(input_ids, image_inputs)
1051 + image_inputs = \
1052 + self._parse_and_validate_inputs(input_ids, **kwargs)
1053 + vlm_embeddings, _ = self.get_embedding_with_vision(
1054 + input_ids, image_inputs)
1055
1056 # always pass the input via `inputs_embeds`
1057 # to make sure the computation graph is consistent
@@ -964,15 +1457,15 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
1457 _SUPPORT_VERSION = {
1458 (2, 0): MiniCPMV2_0,
1459 (2, 5): MiniCPMV2_5,
967 - (2, 6): MiniCPMV2_6
1460 + (2, 6): MiniCPMV2_6,
1461 }
1462
1463
971 -@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_minicpmv)
972 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_minicpmv_image_tokens)
973 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_minicpmv)
974 -@INPUT_REGISTRY.register_input_processor(input_processor_for_minicpmv)
975 -class MiniCPMV(MiniCPMVBaseModel, SupportsLoRA):
1464 +@MULTIMODAL_REGISTRY.register_processor(
1465 + MiniCPMVMultiModalProcessor,
1466 + info=MiniCPMVProcessingInfo,
1467 + dummy_inputs=MiniCPMVDummyInputsBuilder)
1468 +class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA):
1469 """
1470 Different versions of MiniCPMV use different visual encoders and LLMs,
1471 which is not conducive to the current integration logic of LoRA and
vllm/model_executor/models/registry.py
+1
@@ -162,6 +162,7 @@ _MULTIMODAL_MODELS = {
162 "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"), # noqa: E501
163 "LlavaOnevisionForConditionalGeneration": ("llava_onevision", "LlavaOnevisionForConditionalGeneration"), # noqa: E501
164 "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"), # noqa: E501
165 + "MiniCPMO": ("minicpmo", "MiniCPMO"),
166 "MiniCPMV": ("minicpmv", "MiniCPMV"),
167 "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
168 "NVLM_D": ("nvlm_d", "NVLM_D_Model"),