[Bugfix][MM] Fix MiniCPM-o image processor reuse on Transformers v5 (#54501)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Andreas Karatzas committed
Aug 31, 2026 at 02:07 UTC
399247cc8877f60f02f3aa859c61c3330a59bfbb
3 files changed
+41
-23
tests/models/multimodal/processing/test_minicpmv.py
+27
-11
@@ -7,6 +7,7 @@ import numpy as np
7
import pytest
8
9
from vllm.multimodal import MULTIMODAL_REGISTRY
10
+from vllm.multimodal.parse import ImageSize
11
12
from ...utils import build_model_context
13
@@ -27,21 +28,36 @@ def test_get_hf_processor_for_same_model_different_kwargs(model_id: str):
28
29
30
@pytest.mark.parametrize(
30
- "model_ids", [("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4")]
31
+ "model_ids",
32
+ [
33
+ ("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4"),
34
+ ("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-o-2_6"),
35
+ ],
36
)
32
-def test_image_processor_for_dif_model(model_ids):
33
- model_id_25, model_id_4 = model_ids
37
+def test_image_processor_for_different_models(model_ids):
38
+ first_model_id, second_model_id = model_ids
39
35
- ctx_25 = build_model_context(model_id_25, limit_mm_per_prompt={"image": 1})
36
- processor_25 = MULTIMODAL_REGISTRY.create_processor(ctx_25.model_config)
37
- image_processor_25 = processor_25.info.get_image_processor()
40
+ first_ctx = build_model_context(
41
+ first_model_id,
42
+ limit_mm_per_prompt={"image": 1},
43
+ )
44
+ first_processor = MULTIMODAL_REGISTRY.create_processor(first_ctx.model_config)
45
+ first_image_processor = first_processor.info.get_image_processor()
46
+
47
+ second_ctx = build_model_context(
48
+ second_model_id,
49
+ limit_mm_per_prompt={"image": 1},
50
+ )
51
+ second_processor = MULTIMODAL_REGISTRY.create_processor(second_ctx.model_config)
52
+ second_image_processor = second_processor.info.get_image_processor()
53
39
- ctx_4 = build_model_context(model_id_4, limit_mm_per_prompt={"image": 1})
40
- processor_4 = MULTIMODAL_REGISTRY.create_processor(ctx_4.model_config)
41
- image_processor_4 = processor_4.info.get_image_processor()
54
+ second_processor.info.get_sliced_grid(ImageSize(width=128, height=128))
55
43
- assert type(image_processor_25) is not type(image_processor_4)
44
- assert type(image_processor_25).__module__ != type(image_processor_4).__module__
56
+ assert type(first_image_processor) is not type(second_image_processor)
57
+ assert (
58
+ type(first_image_processor).__module__
59
+ != type(second_image_processor).__module__
60
+ )
61
62
63
@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
vllm/model_executor/models/minicpmo.py
+6
-6
@@ -282,28 +282,28 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
282
def get_hf_processor(self, **kwargs: object) -> "MiniCPMOProcessor":
283
"""Get vendored MiniCPMOProcessor for multimodal (image+audio) inputs.
284
285
- Creates a vendored processor that reuses the HF image processor,
286
- feature extractor, and tokenizer; applies the correct audio pooling
287
- configuration; and converts numpy arrays in the image processor to
288
- lists for serialization compatibility. The returned processor is
285
+ Creates a vendored processor that uses the checkpoint-specific HF image
286
+ processor, feature extractor, and tokenizer; applies the correct audio
287
+ pooling configuration; and converts numpy arrays in the image processor
288
+ to lists for serialization compatibility. The returned processor is
289
compatible with Transformers v5.
290
"""
291
import numpy as np
292
293
hf_processor = self.ctx.get_hf_processor(**kwargs)
294
+ image_processor = self._get_checkpoint_image_processor(**kwargs)
295
296
from vllm.transformers_utils.processors.minicpmo import MiniCPMOProcessor
297
298
# Create vendored processor with correct configuration
299
vendored_processor = MiniCPMOProcessor(
299
- image_processor=hf_processor.image_processor,
300
+ image_processor=image_processor,
301
feature_extractor=hf_processor.feature_extractor,
302
tokenizer=hf_processor.tokenizer,
303
pool_step=self.get_default_audio_pool_step(),
304
)
305
306
# Convert numpy arrays in image processor to lists for serialization
306
- image_processor = vendored_processor.image_processor
307
for attr in ("mean", "std"):
308
val = getattr(image_processor, attr, None)
309
if val is not None and isinstance(val, np.ndarray):
vllm/model_executor/models/minicpmv.py
+8
-6
@@ -611,16 +611,12 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
611
def get_hf_config(self):
612
return self.ctx.get_hf_config()
613
614
- def get_hf_processor(self, **kwargs: object):
614
+ def _get_checkpoint_image_processor(self, **kwargs: object):
615
model_config = self.ctx.model_config
616
processor_cls = self._image_processor_cls
617
merged_kwargs = _merge_mm_kwargs(model_config, processor_cls, **kwargs)
618
619
- # AutoProcessor only for tokenizer; its image_processor is resolved by
620
- # class name and can pick the wrong checkpoint across MiniCPM-V versions.
621
- hf_processor = self.ctx.get_hf_processor(**kwargs)
622
-
623
- image_processor = cached_get_image_processor(
619
+ return cached_get_image_processor(
620
model_config.model,
621
revision=model_config.revision,
622
trust_remote_code=model_config.trust_remote_code,
@@ -628,6 +624,12 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
624
**merged_kwargs,
625
)
626
627
+ def get_hf_processor(self, **kwargs: object):
628
+ # AutoProcessor only for tokenizer; its image_processor is resolved by
629
+ # class name and can pick the wrong MiniCPM checkpoint.
630
+ hf_processor = self.ctx.get_hf_processor(**kwargs)
631
+ image_processor = self._get_checkpoint_image_processor(**kwargs)
632
+
633
from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor
634
635
vendored_processor = MiniCPMVProcessor(