[Bugfix][MM] Fix MiniCPM-V placeholder replacement and image processor loading on Transformers v5 (#48413)
Signed-off-by: YunzhuLu <lucia.yunzhu@gmail.com> Signed-off-by: Yunzhu Lu <lucia.yunzhu@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Yunzhu Lu committed
Aug 6, 2026 at 12:12 UTC
f84df12c6ac30fb96a24491bb9846b28f04cf24c
4 files changed
+218
-10
tests/models/multimodal/processing/test_minicpmv.py
new
+74
@@ -0,0 +1,74 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+"""Tests for MiniCPMV's multimodal preprocessing."""
5
+
6
+import numpy as np
7
+import pytest
8
+
9
+from vllm.multimodal import MULTIMODAL_REGISTRY
10
+
11
+from ...utils import build_model_context
12
+
13
+
14
+@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
15
+def test_get_hf_processor_for_same_model_different_kwargs(model_id: str):
16
+ """Calls with different kwargs must not reuse stale processor instances."""
17
+ ctx = build_model_context(
18
+ model_id,
19
+ limit_mm_per_prompt={"image": 1},
20
+ )
21
+ processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
22
+ info = processor.info
23
+ processor_1 = info.get_hf_processor(max_slice_nums=1)
24
+ processor_2 = info.get_hf_processor(max_slice_nums=2)
25
+ assert processor_1.image_processor.max_slice_nums == 1
26
+ assert processor_2.image_processor.max_slice_nums == 2
27
+
28
+
29
+@pytest.mark.parametrize(
30
+ "model_ids", [("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4")]
31
+)
32
+def test_image_processor_for_dif_model(model_ids):
33
+ model_id_25, model_id_4 = model_ids
34
+
35
+ ctx_25 = build_model_context(model_id_25, limit_mm_per_prompt={"image": 1})
36
+ processor_25 = MULTIMODAL_REGISTRY.create_processor(ctx_25.model_config)
37
+ image_processor_25 = processor_25.info.get_image_processor()
38
+
39
+ ctx_4 = build_model_context(model_id_4, limit_mm_per_prompt={"image": 1})
40
+ processor_4 = MULTIMODAL_REGISTRY.create_processor(ctx_4.model_config)
41
+ image_processor_4 = processor_4.info.get_image_processor()
42
+
43
+ assert type(image_processor_25) is not type(image_processor_4)
44
+ assert type(image_processor_25).__module__ != type(image_processor_4).__module__
45
+
46
+
47
+@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
48
+def test_prompt_has_dif_BPE_boundaries_in_context(model_id: str):
49
+ ctx = build_model_context(
50
+ model_id,
51
+ limit_mm_per_prompt={"image": 1},
52
+ )
53
+ processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
54
+ tokenizer = ctx.get_tokenizer()
55
+
56
+ messages = [
57
+ {"role": "user", "content": "(<image>./</image>)\nWhat is in this image?"}
58
+ ]
59
+ prompt = tokenizer.apply_chat_template(
60
+ messages,
61
+ tokenize=False,
62
+ add_generation_prompt=True,
63
+ )
64
+ image = np.zeros((768, 1024, 3), dtype=np.uint8)
65
+
66
+ mm_items = processor.info.parse_mm_data({"image": [image]})
67
+ processed = processor(
68
+ prompt,
69
+ mm_items=mm_items,
70
+ hf_processor_mm_kwargs={},
71
+ )
72
+ image_placeholders = processed["mm_placeholders"].get("image", [])
73
+ assert len(image_placeholders) == 1
74
+ assert image_placeholders[0].length > 0
tests/models/registry.py
-7
@@ -1107,13 +1107,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1107
"4.0": "openbmb/MiniCPM-V-4",
1108
"4.5": "openbmb/MiniCPM-V-4_5",
1109
},
1110
- max_transformers_version="4.57",
1111
- transformers_version_reason={
1112
- "vllm": (
1113
- "MiniCPMVBatchFeature is incompatible with its base class in "
1114
- "Transformers v5. See https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/discussions/78"
1115
- )
1116
- },
1110
trust_remote_code=True,
1111
),
1112
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
vllm/model_executor/models/minicpmv.py
+111
-3
@@ -27,7 +27,7 @@
27
import math
28
from collections import defaultdict
29
from collections.abc import Callable, Iterable, Mapping, Sequence, Set
30
-from functools import partial
30
+from functools import cached_property, partial
31
from itertools import chain
32
from typing import Annotated, Any, Literal, TypeAlias
33
@@ -37,6 +37,10 @@ import torch.types
37
from torch import nn
38
from torch.nn.init import trunc_normal_
39
from transformers import BatchFeature, PretrainedConfig
40
+from transformers.dynamic_module_utils import (
41
+ get_class_from_dynamic_module,
42
+ resolve_trust_remote_code,
43
+)
44
from typing_extensions import TypeVar
45
46
from vllm.config import VllmConfig
@@ -74,6 +78,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder
78
from vllm.multimodal.processing.processor import (
79
BaseMultiModalProcessor,
80
BaseProcessingInfo,
81
+ MultiModalPromptUpdates,
82
+ PlaceholderFeaturesInfo,
83
PromptReplacement,
84
PromptUpdate,
85
PromptUpdateDetails,
@@ -82,6 +88,11 @@ from vllm.multimodal.processing.processor import (
88
)
89
from vllm.platforms import current_platform
90
from vllm.sequence import IntermediateTensors
91
+from vllm.transformers_utils.processor import (
92
+ _merge_mm_kwargs,
93
+ cached_get_image_processor,
94
+)
95
+from vllm.transformers_utils.utils import convert_model_repo_to_path
96
from vllm.utils.collection_utils import flatten_2d_lists
97
from vllm.utils.tensor_schema import TensorSchema, TensorShape
98
from vllm.utils.torch_utils import set_default_torch_dtype
@@ -559,16 +570,63 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
570
image_pattern = "(<image>./</image>)"
571
video_pattern = "(<video>./</video>)"
572
573
+ @cached_property
574
+ def _image_processor_cls(self):
575
+ model_config = self.ctx.model_config
576
+ model_path = convert_model_repo_to_path(model_config.model)
577
+
578
+ from transformers import ImageProcessingMixin
579
+
580
+ image_processor_config, _ = ImageProcessingMixin.get_image_processor_dict(
581
+ model_path,
582
+ revision=model_config.revision,
583
+ token=model_config.hf_token,
584
+ )
585
+
586
+ auto_map = image_processor_config.get("auto_map") or {}
587
+ class_ref = auto_map.get("AutoImageProcessor")
588
+ if not class_ref:
589
+ raise ValueError(
590
+ "Missing auto_map['AutoImageProcessor'] in image processor config "
591
+ f"for {model_config.model!r}"
592
+ )
593
+
594
+ resolve_trust_remote_code(
595
+ model_config.trust_remote_code,
596
+ model_config.model,
597
+ has_local_code=False,
598
+ has_remote_code=True,
599
+ )
600
+ return get_class_from_dynamic_module(
601
+ class_ref,
602
+ model_config.model,
603
+ revision=model_config.revision,
604
+ )
605
+
606
def get_hf_config(self):
607
return self.ctx.get_hf_config()
608
609
def get_hf_processor(self, **kwargs: object):
610
+ model_config = self.ctx.model_config
611
+ processor_cls = self._image_processor_cls
612
+ merged_kwargs = _merge_mm_kwargs(model_config, processor_cls, **kwargs)
613
+
614
+ # AutoProcessor only for tokenizer; its image_processor is resolved by
615
+ # class name and can pick the wrong checkpoint across MiniCPM-V versions.
616
hf_processor = self.ctx.get_hf_processor(**kwargs)
617
618
+ image_processor = cached_get_image_processor(
619
+ model_config.model,
620
+ revision=model_config.revision,
621
+ trust_remote_code=model_config.trust_remote_code,
622
+ processor_cls_overrides=processor_cls,
623
+ **merged_kwargs,
624
+ )
625
+
626
from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor
627
628
vendored_processor = MiniCPMVProcessor(
571
- image_processor=hf_processor.image_processor,
629
+ image_processor=image_processor,
630
tokenizer=hf_processor.tokenizer,
631
version=self.get_model_version(),
632
)
@@ -576,7 +634,6 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
634
635
# NumPy arrays are considered as Iterable but not Sequence in
636
# https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
579
- image_processor = hf_processor.image_processor # type: ignore
637
# transformers v5+ renamed `mean`/`std` -> `image_mean`/`image_std`
638
for attr in ("mean", "std", "image_mean", "image_std"):
639
val = getattr(image_processor, attr, None)
@@ -863,6 +920,57 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
920
**self.process_videos(mm_data, mm_kwargs, tok_kwargs),
921
}
922
923
+ def _apply_prompt_updates(
924
+ self,
925
+ token_ids: list[int],
926
+ mm_prompt_updates: MultiModalPromptUpdates,
927
+ ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
928
+ """Apply multi-modal prompt updates to token IDs."""
929
+ tokenizer = self.info.get_tokenizer()
930
+
931
+ new_token_ids, match_result = self._apply_token_matches(
932
+ token_ids,
933
+ mm_prompt_updates,
934
+ )
935
+
936
+ # If the search text does not represent a special token,
937
+ # it may have different token IDs in the prompt, because
938
+ # the tokens may go across the boundaries of the search text.
939
+ # ----
940
+ # e.g. when searching for "foo" in "food", if "food" itself makes
941
+ # up a token, then the token ID of "foo" will not appear at all
942
+ # ----
943
+ # Since it is inefficient to search for all possible tokenizations
944
+ # of the search text in the prompt, we instead perform string-based
945
+ # updates on the decoded token IDs, then encode them back.
946
+ if not all(
947
+ all(update_idx is not None for update_idx in update_idxs)
948
+ for update_idxs in match_result.values()
949
+ ):
950
+ new_token_ids, match_result = self._apply_text_matches_as_segmented_tokens(
951
+ _seq2text(tokenizer, token_ids, use_cache=False),
952
+ mm_prompt_updates,
953
+ )
954
+
955
+ matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
956
+ for modality, update_idxs in match_result.items():
957
+ for item_idx, update_idx in enumerate(update_idxs):
958
+ assert update_idx is not None, (
959
+ "Failed to apply prompt replacement for "
960
+ f"mm_items[{modality!r}][{item_idx}]"
961
+ )
962
+
963
+ matched_updates[modality].append(
964
+ [mm_prompt_updates[modality][item_idx][update_idx]]
965
+ )
966
+
967
+ placeholders = self._find_mm_placeholders(
968
+ new_token_ids,
969
+ dict(matched_updates),
970
+ )
971
+
972
+ return new_token_ids, placeholders
973
+
974
def _base_call_hf_processor(
975
self,
976
prompts: list[str],
vllm/multimodal/processing/processor.py
+33
@@ -903,6 +903,29 @@ def apply_text_matches(
903
return "".join(texts), result
904
905
906
+def apply_text_matches_as_segmented_tokens(
907
+ prompt: str,
908
+ mm_prompt_updates: "MultiModalPromptUpdates",
909
+ tokenizer: TokenizerLike | None,
910
+) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
911
+ """
912
+ Apply the updates in `mm_prompt_updates` to `prompt`.
913
+
914
+ Matches are exclusive even when multiple modalities share
915
+ the same placeholder tokens. In that case, the modality that
916
+ appears earlier in `mm_prompt_updates` takes priority.
917
+
918
+ Each segment is encoded separately instead of being joined into one
919
+ string and encoded in a single pass. Joining first would let BPE merge
920
+ tokens across a segment boundary, silently change how a text
921
+ (non-special-token) placeholder is tokenized.
922
+ """
923
+ texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
924
+ token_id_seqs = [_seq2tokens(tokenizer, text, use_cache=False) for text in texts]
925
+
926
+ return flatten_2d_lists(token_id_seqs), result
927
+
928
+
929
def _iter_placeholders(
930
prompt: list[int],
931
mm_prompt_updates: "MultiModalPromptUpdates",
@@ -1584,6 +1607,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1607
tokenizer = self.info.get_tokenizer()
1608
return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
1609
1610
+ def _apply_text_matches_as_segmented_tokens(
1611
+ self,
1612
+ prompt: str,
1613
+ mm_prompt_updates: MultiModalPromptUpdates,
1614
+ ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
1615
+ tokenizer = self.info.get_tokenizer()
1616
+ return apply_text_matches_as_segmented_tokens(
1617
+ prompt, mm_prompt_updates, tokenizer
1618
+ )
1619
+
1620
def _apply_prompt_updates(
1621
self,
1622
token_ids: list[int],