A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 17:45 UTC
[Model] Support MiniCPM-o 4.5 (#33431)
Signed-off-by: caitianchi <caitianchi@modelbest.cn> Signed-off-by: tc-mb <caitianchi@modelbest.cn> Co-authored-by: mslv <mslv@baai.ac.cn>
tc-mb committed
Feb 6, 2026 at 23:29 UTC
4707f7ebb4f10f42a951875ff35e1959caa17526
1 file changed
+86
-7
vllm/model_executor/models/minicpmo.py
+86
-7
@@ -64,6 +64,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
64
from .minicpmv import (
65
_MAX_FRAMES_PER_VIDEO,
66
MiniCPMV2_6,
67
+ MiniCPMV4_5,
68
MiniCPMVDummyInputsBuilder,
69
MiniCPMVMultiModalDataParser,
70
MiniCPMVMultiModalProcessor,
@@ -197,7 +198,9 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
198
)
199
200
def get_default_audio_pool_step(self) -> int:
200
- return 2
201
+ hf_config = self.get_hf_config()
202
+ # MiniCPM-o 4.5 uses pool_step=5, older versions use 2
203
+ return getattr(hf_config, "audio_pool_step", 2)
204
205
def get_default_audio_sampling_rate(self) -> int:
206
return 16000
@@ -521,12 +524,9 @@ class MiniCPMWhisperEncoder(WhisperEncoder):
524
)
525
526
524
-@MULTIMODAL_REGISTRY.register_processor(
525
- MiniCPMOMultiModalProcessor,
526
- info=MiniCPMOProcessingInfo,
527
- dummy_inputs=MiniCPMODummyInputsBuilder,
528
-)
529
-class MiniCPMO(MiniCPMV2_6):
527
+class MiniCPMOBaseModel:
528
+ """Base mixin class for MiniCPM-O models with audio support."""
529
+
530
packed_modules_mapping = {
531
"qkv_proj": [
532
"q_proj",
@@ -767,3 +767,82 @@ class MiniCPMO(MiniCPMV2_6):
767
multimodal_embeddings += tuple(audio_embeddings)
768
769
return multimodal_embeddings
770
+
771
+
772
+class MiniCPMO2_6(MiniCPMOBaseModel, MiniCPMV2_6):
773
+ """MiniCPM-O 2.6 model with Qwen2 backbone."""
774
+
775
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
776
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
777
+
778
+ with self._mark_tower_model(vllm_config, "audio"):
779
+ self.apm = self.init_audio_module(
780
+ vllm_config=vllm_config, prefix=maybe_prefix(prefix, "apm")
781
+ )
782
+
783
+
784
+class MiniCPMO4_5(MiniCPMOBaseModel, MiniCPMV4_5):
785
+ """MiniCPM-O 4.5 model with Qwen3 backbone."""
786
+
787
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
788
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
789
+
790
+ with self._mark_tower_model(vllm_config, "audio"):
791
+ self.apm = self.init_audio_module(
792
+ vllm_config=vllm_config, prefix=maybe_prefix(prefix, "apm")
793
+ )
794
+
795
+
796
+_MINICPMO_SUPPORT_VERSION = {
797
+ (2, 6): MiniCPMO2_6,
798
+ (4, 5): MiniCPMO4_5,
799
+}
800
+
801
+
802
+@MULTIMODAL_REGISTRY.register_processor(
803
+ MiniCPMOMultiModalProcessor,
804
+ info=MiniCPMOProcessingInfo,
805
+ dummy_inputs=MiniCPMODummyInputsBuilder,
806
+)
807
+class MiniCPMO(MiniCPMOBaseModel, MiniCPMV2_6):
808
+ """
809
+ MiniCPM-O model with audio support.
810
+ Different versions use different LLM backbones:
811
+ - Version 2.6: Uses Qwen2
812
+ - Version 4.5: Uses Qwen3
813
+ """
814
+
815
+ def __new__(cls, *, vllm_config: VllmConfig, prefix: str = ""):
816
+ config = vllm_config.model_config.hf_config
817
+
818
+ # Determine version from config
819
+ if hasattr(config, "version"):
820
+ try:
821
+ version_str = str(config.version)
822
+ version_parts = version_str.split(".")
823
+ version = tuple(int(x) for x in version_parts[:2])
824
+ except (ValueError, TypeError) as e:
825
+ raise ValueError(
826
+ f"Invalid model version format in config: {config.version}. "
827
+ "Expected a dot-separated version string like '4.5'."
828
+ ) from e
829
+ else:
830
+ # Default to 2.6 for backward compatibility
831
+ version = (2, 6)
832
+
833
+ # Dispatch class based on version
834
+ instance_cls = _MINICPMO_SUPPORT_VERSION.get(version)
835
+ if instance_cls is None:
836
+ supported_versions = ", ".join(
837
+ [f"{v[0]}.{v[1]}" for v in sorted(_MINICPMO_SUPPORT_VERSION.keys())]
838
+ )
839
+ raise ValueError(
840
+ f"Currently, MiniCPMO only supports versions "
841
+ f"{supported_versions}. Got version: {version}"
842
+ )
843
+
844
+ return instance_cls(vllm_config=vllm_config, prefix=prefix)
845
+
846
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
847
+ # This __init__ won't be called due to __new__ returning a different class
848
+ pass