@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7a9cc5e7f0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 3, 20:15 UTC

[Model] Support MiniCPM-V 4.6 (#41254)

Signed-off-by: caitianchi <caitianchi@tc-mb.com> Signed-off-by: tc-mb <157115220+tc-mb@users.noreply.github.com> Co-authored-by: caitianchi <caitianchi@tc-mb.com>

tc-mb committed May 12, 2026 at 22:28 UTC 7a9cc5e7f0cd8b06da9fc7503a442d86cad097d2
5 files changed +1302 -5
tests/models/registry.py
+4
@@ -1098,6 +1098,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1098 },
1099 trust_remote_code=True,
1100 ),
1101 + "MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
1102 + "openbmb/MiniCPM-V-4_6",
1103 + min_transformers_version="5.7.0",
1104 + ),
1105 "MiniMaxVL01ForConditionalGeneration": _HfExamplesInfo(
1106 "MiniMaxAI/MiniMax-VL-01",
1107 trust_remote_code=True,
vllm/model_executor/models/minicpmv.py
+88 -5
@@ -546,8 +546,9 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
546 # NumPy arrays are considered as Iterable but not Sequence in
547 # https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
548 image_processor = hf_processor.image_processor # type: ignore
549 - for attr in ("mean", "std"):
550 - val = getattr(image_processor, attr)
549 + # transformers v5+ renamed `mean`/`std` -> `image_mean`/`image_std`
550 + for attr in ("mean", "std", "image_mean", "image_std"):
551 + val = getattr(image_processor, attr, None)
552 if isinstance(val, np.ndarray):
553 setattr(image_processor, attr, val.tolist())
554
@@ -585,6 +586,50 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
586 if version == (2, 0) or version == (2, 5):
587 return image_processor.get_slice_image_placeholder(image_size)
588
589 + if version == (4, 6):
590 + if max_slice_nums is None:
591 + max_slice_nums = image_processor.max_slice_nums
592 + grids = image_processor.get_sliced_grid(
593 + image_size,
594 + max_slice_nums=max_slice_nums,
595 + )
596 + patch_size = image_processor.patch_size
597 + scale_resolution = image_processor.scale_resolution
598 +
599 + allow_upscale = grids is None
600 + best_size = image_processor.find_best_resize(
601 + image_size,
602 + scale_resolution,
603 + patch_size,
604 + allow_upscale=allow_upscale,
605 + )
606 + h_patches = best_size[1] // patch_size
607 + w_patches = best_size[0] // patch_size
608 + source_image_visual_tokens = (h_patches // 4) * (w_patches // 4)
609 +
610 + if grids is not None:
611 + refine_size = image_processor.get_refine_size(
612 + image_size,
613 + grids,
614 + scale_resolution,
615 + patch_size,
616 + allow_upscale=True,
617 + )
618 + pw = refine_size[0] // grids[0]
619 + ph = refine_size[1] // grids[1]
620 + patch_visual_tokens = (ph // patch_size // 4) * (pw // patch_size // 4)
621 + else:
622 + patch_visual_tokens = source_image_visual_tokens
623 +
624 + return image_processor.get_slice_image_placeholder(
625 + grids if grids is not None else [0, 0],
626 + image_idx=image_idx,
627 + max_slice_nums=max_slice_nums,
628 + use_image_id=use_image_id,
629 + source_image_visual_tokens=source_image_visual_tokens,
630 + patch_visual_tokens=patch_visual_tokens,
631 + )
632 +
633 return image_processor.get_slice_image_placeholder(
634 image_size,
635 image_idx=image_idx,
@@ -618,11 +663,44 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
663 max_slice_nums: int | None = None,
664 ) -> int:
665 image_processor = self.get_image_processor()
666 + version = self.get_model_version()
667
668 grid = self.get_sliced_grid(
669 image_size,
670 max_slice_nums=max_slice_nums,
671 )
672 +
673 + if version == (4, 6):
674 + patch_size = image_processor.patch_size
675 + scale_resolution = image_processor.scale_resolution
676 +
677 + allow_upscale = grid is None
678 + best_size = image_processor.find_best_resize(
679 + image_size,
680 + scale_resolution,
681 + patch_size,
682 + allow_upscale=allow_upscale,
683 + )
684 + h_p = best_size[1] // patch_size
685 + w_p = best_size[0] // patch_size
686 + source_tokens = (h_p // 4) * (w_p // 4)
687 +
688 + if grid is None:
689 + return source_tokens
690 +
691 + refine_size = image_processor.get_refine_size(
692 + image_size,
693 + grid,
694 + scale_resolution,
695 + patch_size,
696 + allow_upscale=True,
697 + )
698 + pw = refine_size[0] // grid[0]
699 + ph = refine_size[1] // grid[1]
700 + patch_tokens = (ph // patch_size // 4) * (pw // patch_size // 4)
701 + ncols, nrows = grid
702 + return source_tokens + ncols * nrows * patch_tokens
703 +
704 if grid is None:
705 ncols = nrows = 0
706 else:
@@ -839,7 +917,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
917 out_keys: set[str],
918 ) -> dict[str, NestedTensors]:
919 # This processor supports zipping prompt and mm_data together
842 - if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5)}:
920 + if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5), (4, 6)}:
921 inputs = super()._call_hf_processor(
922 prompt=prompts, # type: ignore
923 mm_data=mm_data,
@@ -971,10 +1049,15 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
1049 if version == (2, 0) or version == (2, 5):
1050 im_start = image_processor.im_start_token
1051 im_end = image_processor.im_end_token
974 - else:
1052 + elif hasattr(image_processor, "im_id_start"):
1053 im_start = image_processor.im_id_start
1054 im_end = image_processor.im_id_end
1055 + else:
1056 + # transformers v5.7+ keeps im_id tokens on the tokenizer.
1057 + im_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
1058 + im_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
1059
1060 + embed_text = getattr(tokenizer, "image_token", "<unk>")
1061 new_update = new_update.with_content(
1062 PromptUpdateDetails.select_text(
1063 text.replace(
@@ -982,7 +1065,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
1065 f"{im_start}{new_item_idx}{im_end}",
1066 1,
1067 ),
985 - "<unk>",
1068 + embed_text,
1069 )
1070 )
1071
vllm/model_executor/models/minicpmv4_6.py new
+1205
@@ -0,0 +1,1205 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Inference-only MiniCPM-V 4.6 model (MiniCPMV4_6ForConditionalGeneration)."""
4 +
5 +from collections.abc import Iterable, Mapping
6 +from typing import Any
7 +
8 +import torch
9 +from torch import nn
10 +from transformers import MiniCPMV4_6Config
11 +
12 +from vllm.config import VllmConfig
13 +from vllm.distributed import get_tensor_model_parallel_world_size
14 +from vllm.model_executor.layers.activation import get_act_fn
15 +from vllm.model_executor.layers.attention import MMEncoderAttention
16 +from vllm.model_executor.layers.linear import (
17 + QKVParallelLinear,
18 + RowParallelLinear,
19 +)
20 +from vllm.model_executor.layers.mamba.mamba_utils import (
21 + MambaStateCopyFuncCalculator,
22 + MambaStateDtypeCalculator,
23 + MambaStateShapeCalculator,
24 +)
25 +from vllm.model_executor.layers.quantization import QuantizationConfig
26 +from vllm.multimodal import MULTIMODAL_REGISTRY
27 +from vllm.multimodal.inputs import (
28 + MultiModalFeatureSpec,
29 + MultiModalFieldConfig,
30 + NestedTensors,
31 +)
32 +from vllm.multimodal.parse import ImageProcessorItems, VideoProcessorItems
33 +from vllm.multimodal.processing.processor import (
34 + PromptReplacement,
35 + PromptUpdateDetails,
36 +)
37 +from vllm.sequence import IntermediateTensors
38 +
39 +from .idefics2_vision_model import Idefics2VisionTransformer
40 +from .interfaces import (
41 + HasInnerState,
42 + IsHybrid,
43 + MultiModalEmbeddings,
44 + SupportsMRoPE,
45 + SupportsMultiModal,
46 + SupportsPP,
47 + _require_is_multimodal,
48 +)
49 +from .minicpmv import (
50 + MiniCPMVDummyInputsBuilder,
51 + MiniCPMVImageEmbeddingInputs,
52 + MiniCPMVImageEmbeddingItems,
53 + MiniCPMVImagePixelInputs,
54 + MiniCPMVMultiModalProcessor,
55 + MiniCPMVProcessingInfo,
56 + MiniCPMVVideoEmbeddingItems,
57 +)
58 +from .module_mapping import MultiModelKeys
59 +from .qwen3_5 import Qwen3_5ForCausalLM
60 +from .utils import (
61 + AutoWeightsLoader,
62 + WeightsMapper,
63 + _merge_multimodal_embeddings,
64 + flatten_bn,
65 + maybe_prefix,
66 +)
67 +from .vision import is_vit_use_data_parallel
68 +
69 +
70 +def _minicpmv4_6_field_config(hf_inputs: Mapping[str, torch.Tensor]):
71 + fields = dict(
72 + pixel_values=MultiModalFieldConfig.batched("image"),
73 + tgt_sizes=MultiModalFieldConfig.batched("image"),
74 + image_embeds=MultiModalFieldConfig.batched("image"),
75 + video_pixel_values=MultiModalFieldConfig.batched("video"),
76 + video_image_sizes=MultiModalFieldConfig.batched("video"),
77 + video_tgt_sizes=MultiModalFieldConfig.batched("video"),
78 + video_embeds=MultiModalFieldConfig.batched("video"),
79 + )
80 + if "use_vit_merger" in hf_inputs:
81 + fields["use_vit_merger"] = MultiModalFieldConfig.batched("image")
82 + return fields
83 +
84 +
85 +class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
86 + def _resolve_downsample_mode(
87 + self,
88 + mm_kwargs: Mapping[str, object],
89 + ) -> str:
90 + ds = mm_kwargs.get("downsample_mode")
91 + if ds is not None:
92 + return str(ds)
93 + return self.info._get_downsample_mode()
94 +
95 + def get_image_prompt_texts(
96 + self,
97 + image_size,
98 + image_idx: int = 0,
99 + downsample_mode: str | None = None,
100 + ) -> str:
101 + return self.info.get_slice_image_placeholder(
102 + image_size,
103 + image_idx=image_idx,
104 + downsample_mode=downsample_mode,
105 + )
106 +
107 + def get_video_prompt_texts(
108 + self,
109 + image_size,
110 + num_frames: int,
111 + downsample_mode: str | None = None,
112 + video_idx: int = 0,
113 + ) -> str:
114 + # Match transformers v5.7+ MiniCPMV4_6Processor video formatting:
115 + # <image_id>{video_idx}</image_id>(<image>VIDEO*src</image>
116 + # <slice>VIDEO*patch</slice>...)*num_frames
117 + # Crucially the visual token inside each frame is ``<|video_pad|>``
118 + # (tokenizer.video_token), NOT ``<|image_pad|>`` — they share the same
119 + # embedding-injection role but the language model is conditioned on
120 + # which one is used. Using image_token for video silently produces
121 + # garbage descriptions.
122 + info = self.info
123 + grids, source_tokens, patch_tokens = info._compute_visual_tokens(
124 + image_size,
125 + max_slice_nums=info.get_video_max_slice_num(),
126 + downsample_mode=downsample_mode,
127 + )
128 + tokenizer = info.get_tokenizer()
129 + video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
130 + image_start = getattr(tokenizer, "image_start_token", "<image>")
131 + image_end = getattr(tokenizer, "image_end_token", "</image>")
132 + slice_start = getattr(tokenizer, "slice_start_token", "<slice>")
133 + slice_end = getattr(tokenizer, "slice_end_token", "</slice>")
134 + id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
135 + id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
136 +
137 + per_frame = image_start + video_token * source_tokens + image_end
138 + if grids[0] > 0 and grids[1] > 0 and patch_tokens > 0:
139 + slice_ph = slice_start + video_token * patch_tokens + slice_end
140 + rows = [slice_ph * grids[0] for _ in range(grids[1])]
141 + per_frame += "\n".join(rows)
142 +
143 + body = per_frame * num_frames
144 + return f"{id_start}{video_idx}{id_end}" + body
145 +
146 + def process_images(
147 + self,
148 + mm_data: Mapping[str, object],
149 + mm_kwargs: Mapping[str, object],
150 + tok_kwargs: Mapping[str, object],
151 + ) -> Mapping[str, NestedTensors]:
152 + if (images := mm_data.get("images")) is None:
153 + return {}
154 +
155 + mm_items = self.info.parse_mm_data({"image": images}, validate=False)
156 + parsed_images = mm_items.get_items(
157 + "image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems)
158 + )
159 +
160 + if isinstance(parsed_images, MiniCPMVImageEmbeddingItems):
161 + return {}
162 +
163 + # transformers v5.7+ MiniCPMV4_6ImageProcessor returns
164 + # `pixel_values` (1, C, P, sum_W) where all slices are fused along W
165 + # (NaViT-style), and `target_sizes` (n_slices, 2). vLLM expects each
166 + # image entry to be a 4D tensor (n_slices, C, P, L_max_padded).
167 + n_images = len(parsed_images)
168 + image_processor = self.info.get_image_processor()
169 + patch_size = image_processor.patch_size
170 + per_image_pixel_values: list[torch.Tensor] = []
171 + per_image_tgt_sizes: list[torch.Tensor] = []
172 + for image in parsed_images:
173 + ip_out = image_processor([image], **mm_kwargs)
174 + pv = ip_out["pixel_values"] # (1, C, P, sum_W)
175 + ts = ip_out["target_sizes"] # (n_slices, 2)
176 + if pv.ndim == 4 and pv.shape[0] == 1:
177 + pv = pv.squeeze(0) # (C, P, sum_W)
178 + ts_long = ts.to(torch.long)
179 + split_widths = (ts_long[:, 0] * ts_long[:, 1] * patch_size).tolist()
180 + slices = torch.split(pv, split_widths, dim=-1)
181 + n_slices = len(slices)
182 + l_max = max(s.shape[-1] for s in slices)
183 + out = torch.zeros(
184 + n_slices,
185 + pv.shape[0],
186 + pv.shape[1],
187 + l_max,
188 + dtype=pv.dtype,
189 + device=pv.device,
190 + )
191 + for i, s in enumerate(slices):
192 + out[i, :, :, : s.shape[-1]] = s
193 + per_image_pixel_values.append(out)
194 + per_image_tgt_sizes.append(ts_long)
195 +
196 + image_inputs: dict = {
197 + "pixel_values": per_image_pixel_values,
198 + "tgt_sizes": per_image_tgt_sizes,
199 + }
200 +
201 + ds_mode = self._resolve_downsample_mode(mm_kwargs)
202 + insert_layer_id = getattr(
203 + self.info.get_hf_config(),
204 + "insert_layer_id",
205 + -1,
206 + )
207 + merger_flag = ds_mode != "4x" and insert_layer_id >= 0
208 + image_inputs["use_vit_merger"] = [
209 + torch.tensor([merger_flag], dtype=torch.bool) for _ in range(n_images)
210 + ]
211 + return image_inputs
212 +
213 + def process_videos(
214 + self,
215 + mm_data: Mapping[str, object],
216 + mm_kwargs: Mapping[str, object],
217 + tok_kwargs: Mapping[str, object],
218 + ) -> Mapping[str, NestedTensors]:
219 + if (videos := mm_data.get("videos")) is None:
220 + return {}
221 +
222 + mm_items = self.info.parse_mm_data({"video": videos}, validate=False)
223 + parsed_videos = mm_items.get_items(
224 + "video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems)
225 + )
226 +
227 + if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems):
228 + return {}
229 +
230 + # Treat each video as a sequence of frames. The transformers v5.7+
231 + # `MiniCPMV4_6ImageProcessor` returns NaViT-style fused `pixel_values`;
232 + # we run it per-frame, split the slices, then re-pack each video into
233 + # a single 4D tensor (sum_slices, C, P, L_max_video).
234 + image_processor = self.info.get_image_processor()
235 + patch_size = image_processor.patch_size
236 + video_max_slice = self.info.get_video_max_slice_num()
237 + video_mm_kwargs = {**mm_kwargs, "max_slice_nums": video_max_slice}
238 +
239 + per_video_pixel_values: list[torch.Tensor] = []
240 + per_video_tgt_sizes: list[torch.Tensor] = []
241 +
242 + for video in parsed_videos:
243 + # video is iterable of frames (PIL Image or numpy array).
244 + all_slices: list[torch.Tensor] = []
245 + ts_list: list[torch.Tensor] = []
246 + for frame in video:
247 + ip_out = image_processor([frame], **video_mm_kwargs)
248 + pv = ip_out["pixel_values"] # (1, C, P, sum_W)
249 + ts = ip_out["target_sizes"] # (n_slices, 2)
250 + if pv.ndim == 4 and pv.shape[0] == 1:
251 + pv = pv.squeeze(0) # (C, P, sum_W)
252 + ts_long = ts.to(torch.long)
253 + split_widths = (ts_long[:, 0] * ts_long[:, 1] * patch_size).tolist()
254 + slices = torch.split(pv, split_widths, dim=-1)
255 + all_slices.extend(slices)
256 + ts_list.append(ts_long)
257 +
258 + if not all_slices:
259 + continue
260 +
261 + l_max = max(s.shape[-1] for s in all_slices)
262 + n_total = len(all_slices)
263 + C, P = all_slices[0].shape[0], all_slices[0].shape[1]
264 + out = torch.zeros(
265 + n_total,
266 + C,
267 + P,
268 + l_max,
269 + dtype=all_slices[0].dtype,
270 + device=all_slices[0].device,
271 + )
272 + for i, s in enumerate(all_slices):
273 + out[i, :, :, : s.shape[-1]] = s
274 +
275 + per_video_pixel_values.append(out)
276 + per_video_tgt_sizes.append(torch.cat(ts_list, dim=0))
277 +
278 + if not per_video_pixel_values:
279 + return {}
280 +
281 + return {
282 + "video_pixel_values": per_video_pixel_values,
283 + "video_tgt_sizes": per_video_tgt_sizes,
284 + }
285 +
286 + def _get_prompt_updates(
287 + self,
288 + mm_items,
289 + hf_processor_mm_kwargs: Mapping[str, object],
290 + out_mm_kwargs,
291 + ):
292 + ds_mode = self._resolve_downsample_mode(hf_processor_mm_kwargs)
293 +
294 + placeholders = [
295 + ("image", self.info.image_pattern),
296 + ("video", self.info.video_pattern),
297 + ]
298 + tokenizer = self.info.get_tokenizer()
299 + additional_placeholders = []
300 + for modality, pattern in placeholders:
301 + sub_pattern = tokenizer.decode(
302 + tokenizer.encode(pattern, add_special_tokens=False)
303 + )
304 + if sub_pattern != pattern:
305 + additional_placeholders.append((modality, sub_pattern))
306 + placeholders += additional_placeholders
307 +
308 + # The 4.6 chat_template emits `<|image_pad|>` / `<|video_pad|>` rather
309 + # than `<unk>`, so use those tokens as the embedding selector.
310 + image_embed_text = getattr(tokenizer, "image_token", "<|image_pad|>")
311 + video_embed_text = getattr(tokenizer, "video_token", "<|video_pad|>")
312 +
313 + def get_image_replacement(item_idx: int):
314 + images = mm_items.get_items(
315 + "image",
316 + (MiniCPMVImageEmbeddingItems, ImageProcessorItems),
317 + )
318 + image_size = images.get_image_size(item_idx)
319 + return PromptUpdateDetails.select_text(
320 + self.get_image_prompt_texts(
321 + image_size,
322 + item_idx,
323 + downsample_mode=ds_mode,
324 + ),
325 + image_embed_text,
326 + )
327 +
328 + def get_video_replacement(item_idx: int):
329 + videos = mm_items.get_items(
330 + "video",
331 + (MiniCPMVVideoEmbeddingItems, VideoProcessorItems),
332 + )
333 + frame_size = videos.get_frame_size(item_idx)
334 + num_frames = videos.get_num_frames(item_idx)
335 + return PromptUpdateDetails.select_text(
336 + self.get_video_prompt_texts(
337 + frame_size,
338 + num_frames,
339 + downsample_mode=ds_mode,
340 + video_idx=item_idx,
341 + ),
342 + video_embed_text,
343 + )
344 +
345 + get_replacement = {
346 + "image": get_image_replacement,
347 + "video": get_video_replacement,
348 + }
349 +
350 + return [
351 + PromptReplacement(
352 + modality=modality,
353 + target=pattern,
354 + replacement=get_replacement[modality],
355 + )
356 + for modality, pattern in placeholders
357 + ]
358 +
359 + def _get_mm_fields_config(
360 + self,
361 + hf_inputs,
362 + hf_processor_mm_kwargs: Mapping[str, object],
363 + ) -> Mapping[str, MultiModalFieldConfig]:
364 + return _minicpmv4_6_field_config(hf_inputs)
365 +
366 +
367 +class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo):
368 + # transformers v5.7+ chat_template emits these as image/video placeholders.
369 + image_pattern = "<|image_pad|>"
370 + video_pattern = "<|video_pad|>"
371 +
372 + def get_hf_config(self):
373 + return self.ctx.get_hf_config()
374 +
375 + def _get_expected_hidden_size(self) -> int:
376 + config = self.get_hf_config()
377 + if hasattr(config, "text_config") and config.text_config is not None:
378 + return config.text_config.hidden_size
379 + return config.hidden_size
380 +
381 + def get_model_version(self):
382 + return (4, 6)
383 +
384 + def get_supported_mm_limits(self) -> Mapping[str, int | None]:
385 + return {"image": None, "video": None}
386 +
387 + def get_image_max_slice_num(self) -> int:
388 + config = self.get_hf_config()
389 + if hasattr(config, "slice_config") and config.slice_config is not None:
390 + return getattr(config.slice_config, "max_slice_nums", 9)
391 + return getattr(config, "max_slice_nums", 9)
392 +
393 + def get_video_max_slice_num(self) -> int:
394 + # Override the base class default of 1: transformers v5.7+
395 + # `MiniCPMV4_6VideoProcessor` keeps the same max_slice_nums (default 9)
396 + # as the image processor so that high-res frames get sliced.
397 + try:
398 + hf_processor = self.get_hf_processor()
399 + video_processor = getattr(hf_processor, "video_processor", None)
400 + if video_processor is not None:
401 + return int(getattr(video_processor, "max_slice_nums", 9))
402 + except Exception:
403 + pass
404 + return self.get_image_max_slice_num()
405 +
406 + def _get_downsample_mode(
407 + self,
408 + downsample_mode: str | None = None,
409 + ) -> str:
410 + if downsample_mode is not None:
411 + return downsample_mode
412 + image_processor = self.get_image_processor()
413 + return getattr(image_processor, "downsample_mode", "16x")
414 +
415 + def _compute_visual_tokens(
416 + self,
417 + image_size,
418 + max_slice_nums: int | None = None,
419 + downsample_mode: str | None = None,
420 + ) -> tuple[list[int], int, int]:
421 + """Compute grid, source_image_visual_tokens and patch_visual_tokens.
422 +
423 + Args:
424 + downsample_mode: ``"16x"`` (default, full merge) or ``"4x"``
425 + (skip vit_merger, 4x more visual tokens).
426 +
427 + Returns:
428 + (grids, source_image_visual_tokens, patch_visual_tokens)
429 + grids is [0, 0] when no slicing occurs.
430 + """
431 + image_processor = self.get_image_processor()
432 + if max_slice_nums is None:
433 + max_slice_nums = image_processor.max_slice_nums
434 +
435 + patch_size = image_processor.patch_size
436 + scale_res = image_processor.scale_resolution
437 + downsample_mode = self._get_downsample_mode(downsample_mode)
438 + token_divisor = 4 if downsample_mode == "4x" else 16
439 +
440 + # transformers v5.7+ requires `scale_resolution` arg
441 + try:
442 + grids = image_processor.get_sliced_grid(
443 + image_size,
444 + max_slice_nums,
445 + scale_res,
446 + )
447 + except TypeError:
448 + grids = image_processor.get_sliced_grid(
449 + image_size,
450 + max_slice_nums,
451 + )
452 +
453 + if grids is None:
454 + best_size = image_processor.find_best_resize(
455 + image_size,
456 + scale_res,
457 + patch_size,
458 + allow_upscale=True,
459 + )
460 + source_tokens = (
461 + best_size[0] * best_size[1] // (patch_size * patch_size * token_divisor)
462 + )
463 + return [0, 0], source_tokens, 0
464 +
465 + best_resize = image_processor.find_best_resize(
466 + image_size,
467 + scale_res,
468 + patch_size,
469 + )
470 + source_tokens = (
471 + best_resize[0] * best_resize[1] // (patch_size * patch_size * token_divisor)
472 + )
473 + refine_size = image_processor.get_refine_size(
474 + image_size,
475 + grids,
476 + scale_res,
477 + patch_size,
478 + allow_upscale=True,
479 + )
480 + patch_w = refine_size[0] // grids[0]
481 + patch_h = refine_size[1] // grids[1]
482 + patch_tokens = patch_w * patch_h // (patch_size * patch_size * token_divisor)
483 + return grids, source_tokens, patch_tokens
484 +
485 + def get_slice_image_placeholder(
486 + self,
487 + image_size,
488 + image_idx: int = 0,
489 + max_slice_nums: int | None = None,
490 + use_image_id: bool = True,
491 + downsample_mode: str | None = None,
492 + ) -> str:
493 + grids, source_tokens, patch_tokens = self._compute_visual_tokens(
494 + image_size,
495 + max_slice_nums,
496 + downsample_mode=downsample_mode,
497 + )
498 + image_processor = self.get_image_processor()
499 + # transformers v5.7+ removed `get_slice_image_placeholder` from the
500 + # image_processor and moved the logic into MiniCPMV4_6Processor.
501 + # Replicate it here using tokenizer special tokens.
502 + if hasattr(image_processor, "get_slice_image_placeholder"):
503 + return image_processor.get_slice_image_placeholder(
504 + grids,
505 + image_idx=image_idx,
506 + max_slice_nums=max_slice_nums,
507 + use_image_id=use_image_id,
508 + source_image_visual_tokens=source_tokens,
509 + patch_visual_tokens=patch_tokens,
510 + )
511 + tokenizer = self.get_tokenizer()
512 + image_token = getattr(tokenizer, "image_token", "<|image_pad|>")
513 + image_start = getattr(tokenizer, "image_start_token", "<image>")
514 + image_end = getattr(tokenizer, "image_end_token", "</image>")
515 + slice_start = getattr(tokenizer, "slice_start_token", "<slice>")
516 + slice_end = getattr(tokenizer, "slice_end_token", "</slice>")
517 + id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
518 + id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
519 +
520 + placeholder = image_start + image_token * source_tokens + image_end
521 + if use_image_id:
522 + placeholder = f"{id_start}{image_idx}{id_end}" + placeholder
523 +
524 + num_cols, num_rows = grids[0], grids[1]
525 + if num_cols > 0 and num_rows > 0 and patch_tokens > 0:
526 + slice_ph = slice_start + image_token * patch_tokens + slice_end
527 + slices = [slice_ph * num_cols for _ in range(num_rows)]
528 + placeholder += "\n".join(slices)
529 + return placeholder
530 +
531 + def get_num_image_tokens(
532 + self,
533 + image_size,
534 + max_slice_nums: int | None = None,
535 + downsample_mode: str | None = None,
536 + ) -> int:
537 + grids, source_tokens, patch_tokens = self._compute_visual_tokens(
538 + image_size,
539 + max_slice_nums,
540 + downsample_mode=downsample_mode,
541 + )
542 + return source_tokens + grids[0] * grids[1] * patch_tokens
543 +
544 +
545 +class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module):
546 + def __init__(
547 + self,
548 + config,
549 + quant_config: QuantizationConfig | None = None,
550 + prefix: str = "",
551 + ):
552 + super().__init__()
553 + use_data_parallel = is_vit_use_data_parallel()
554 + self.embed_dim = config.hidden_size
555 + self.num_heads = config.num_attention_heads
556 + self.head_dim = self.embed_dim // self.num_heads
557 + self.scale = self.head_dim**-0.5
558 +
559 + tp_size = 1 if use_data_parallel else get_tensor_model_parallel_world_size()
560 + assert self.num_heads % tp_size == 0
561 + self.num_heads_per_partition = self.num_heads // tp_size
562 +
563 + self.qkv_proj = QKVParallelLinear(
564 + self.embed_dim,
565 + self.head_dim,
566 + self.num_heads,
567 + quant_config=quant_config,
568 + prefix=f"{prefix}.qkv_proj",
569 + disable_tp=use_data_parallel,
570 + )
571 + self.out_proj = RowParallelLinear(
572 + self.embed_dim,
573 + self.embed_dim,
574 + bias=True,
575 + quant_config=quant_config,
576 + prefix=f"{prefix}.out_proj",
577 + disable_tp=use_data_parallel,
578 + )
579 + self.attn = MMEncoderAttention(
580 + self.num_heads_per_partition,
581 + self.head_dim,
582 + self.scale,
583 + prefix=f"{prefix}.attn",
584 + )
585 +
586 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
587 + qkv, _ = self.qkv_proj(hidden_states)
588 + q, k, v = qkv.chunk(3, dim=-1)
589 + attn_out = self.attn(q, k, v)
590 + out, _ = self.out_proj(attn_out)
591 + return out
592 +
593 +
594 +class MiniCPMV4_6ViTWindowAttentionMerger(nn.Module):
595 + def __init__(
596 + self,
597 + config,
598 + quant_config: QuantizationConfig | None = None,
599 + prefix: str = "",
600 + ):
601 + super().__init__()
602 + self.window_kernel_size = (2, 2)
603 + self.embed_dim = config.hidden_size
604 +
605 + self.self_attn = MiniCPMV4_6ViTWindowAttentionSelfAttn(
606 + config,
607 + quant_config=quant_config,
608 + prefix=f"{prefix}.self_attn",
609 + )
610 + self.layer_norm1 = nn.LayerNorm(
611 + self.embed_dim,
612 + eps=config.layer_norm_eps,
613 + )
614 +
615 + hidden_4x = self.embed_dim * 4
616 + inter_4x = config.intermediate_size * 4
617 +
618 + self.pre_norm = nn.LayerNorm(hidden_4x, eps=config.layer_norm_eps)
619 + self.linear_1 = nn.Linear(hidden_4x, inter_4x, bias=True)
620 + self.act = get_act_fn("gelu_pytorch_tanh")
621 + self.linear_2 = nn.Linear(inter_4x, self.embed_dim, bias=True)
622 +
623 + def _apply_window_attention(
624 + self,
625 + valid_states: torch.Tensor,
626 + H: int,
627 + W: int,
628 + ) -> torch.Tensor:
629 + D = valid_states.shape[-1]
630 + wh, ww = self.window_kernel_size
631 + nh, nw = H // wh, W // ww
632 + num_windows = nh * nw
633 +
634 + x = valid_states.view(H, W, D)
635 + x = x.view(nh, wh, nw, ww, D).permute(0, 2, 1, 3, 4).contiguous()
636 + x = x.view(num_windows, wh * ww, D)
637 +
638 + x = self.self_attn(x)
639 +
640 + x = x.view(nh, nw, wh, ww, D).permute(0, 2, 1, 3, 4).contiguous()
641 + return x.view(H * W, D)
642 +
643 + def _apply_mlp_downsample(
644 + self,
645 + valid_states: torch.Tensor,
646 + H: int,
647 + W: int,
648 + ) -> torch.Tensor:
649 + D = valid_states.shape[-1]
650 + wh, ww = self.window_kernel_size
651 + nh, nw = H // wh, W // ww
652 +
653 + x = valid_states.view(H, W, D)
654 + x = x.view(nh, wh, nw, ww, D).permute(0, 2, 1, 3, 4).contiguous()
655 +
656 + residual = x.reshape(nh * nw, wh * ww, D).mean(dim=1)
657 + x = x.reshape(nh * nw, wh * ww * D)
658 +
659 + x = self.pre_norm(x)
660 + x = self.linear_1(x)
661 + x = self.act(x)
662 + x = self.linear_2(x)
663 + return x + residual
664 +
665 + def forward(
666 + self,
667 + hidden_states: torch.Tensor,
668 + tgt_sizes: torch.Tensor,
669 + attention_mask: torch.Tensor | None,
670 + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
671 + B, _L, D = hidden_states.shape
672 + device = hidden_states.device
673 + dtype = hidden_states.dtype
674 +
675 + all_merged = []
676 + new_tgt_sizes = torch.zeros_like(tgt_sizes)
677 +
678 + for b in range(B):
679 + H, W = tgt_sizes[b].tolist()
680 + hs = hidden_states[b, : H * W, :]
681 +
682 + residual = hs
683 + hs = self.layer_norm1(hs)
684 + hs = residual + self._apply_window_attention(hs, H, W)
685 +
686 + wh, ww = self.window_kernel_size
687 + new_H, new_W = H // wh, W // ww
688 + all_merged.append(self._apply_mlp_downsample(hs, H, W))
689 + new_tgt_sizes[b] = torch.tensor(
690 + [new_H, new_W],
691 + device=device,
692 + dtype=tgt_sizes.dtype,
693 + )
694 +
695 + new_num_patches = new_tgt_sizes[:, 0] * new_tgt_sizes[:, 1]
696 + new_max_patches = int(new_num_patches.max().item())
697 + new_hidden = torch.zeros(
698 + B,
699 + new_max_patches,
700 + D,
701 + device=device,
702 + dtype=dtype,
703 + )
704 + for b, merged in enumerate(all_merged):
705 + new_hidden[b, : merged.shape[0], :] = merged
706 +
707 + # Build new attention mask after spatial downsampling
708 + new_attention_mask: torch.Tensor | None = None
709 + if attention_mask is not None:
710 + mask = torch.zeros(
711 + B,
712 + new_max_patches,
713 + dtype=torch.bool,
714 + device=device,
715 + )
716 + for b in range(B):
717 + mask[b, : int(new_num_patches[b].item())] = True
718 + min_val = torch.finfo(dtype).min
719 + new_attention_mask = (~mask).to(dtype=dtype) * min_val
720 + new_attention_mask = new_attention_mask[:, None, None, :]
721 +
722 + return new_hidden, new_tgt_sizes, new_attention_mask
723 +
724 +
725 +class MiniCPMV4_6DownsampleMLP(nn.Module):
726 + """Match HF (transformers v5.7+) parameter naming: pre_norm/linear_1/
727 + act/linear_2 (instead of pre_norm + Sequential(mlp.0/mlp.2))."""
728 +
729 + def __init__(
730 + self,
731 + hidden_size: int,
732 + llm_embed_dim: int,
733 + merge_kernel_size: tuple[int, int] = (2, 2),
734 + ):
735 + super().__init__()
736 + self.merge_kernel_size = merge_kernel_size
737 + self.hidden_size = hidden_size * merge_kernel_size[0] * merge_kernel_size[1]
738 + self.pre_norm = nn.LayerNorm(self.hidden_size, eps=1e-6)
739 + self.linear_1 = nn.Linear(self.hidden_size, self.hidden_size, bias=True)
740 + self.act = get_act_fn("gelu")
741 + self.linear_2 = nn.Linear(self.hidden_size, llm_embed_dim, bias=True)
742 +
743 + def forward(self, x: torch.Tensor) -> torch.Tensor:
744 + x = self.pre_norm(x)
745 + x = self.linear_1(x)
746 + x = self.act(x)
747 + x = self.linear_2(x)
748 + return x
749 +
750 +
751 +class MiniCPMV4_6Merger(nn.Module):
752 + def __init__(
753 + self,
754 + hidden_size: int,
755 + llm_embed_dim: int,
756 + merge_kernel_size: tuple[int, int] = (2, 2),
757 + times: int = 1,
758 + ):
759 + super().__init__()
760 + self.merge_kernel_size = merge_kernel_size
761 + self.times = times
762 + self.mlp = nn.ModuleList(
763 + [
764 + MiniCPMV4_6DownsampleMLP(
765 + hidden_size,
766 + llm_embed_dim if i == times - 1 else hidden_size,
767 + merge_kernel_size,
768 + )
769 + for i in range(times)
770 + ]
771 + )
772 +
773 + def forward(
774 + self,
775 + hidden_states: torch.Tensor,
776 + tgt_sizes: torch.Tensor,
777 + ) -> list[torch.Tensor]:
778 + """
779 + Args:
780 + hidden_states: (B, max_patches, D) padded batch.
781 + tgt_sizes: (B, 2) actual (H, W) per sample.
782 + """
783 + m1, m2 = self.merge_kernel_size
784 + results = []
785 +
786 + for b in range(len(tgt_sizes)):
787 + h, w = tgt_sizes[b].tolist()
788 + n_patches = h * w
789 + hs = hidden_states[b, :n_patches, :]
790 +
791 + hs = hs.reshape(h // m1, m1, w // m2, m2, -1)
792 + hs = hs.permute(0, 2, 1, 3, 4).reshape(
793 + (h // m1) * (w // m2),
794 + m1 * m2 * hs.shape[-1],
795 + )
796 + hs = self.mlp[0](hs)
797 +
798 + if self.times > 1:
799 + cur_h, cur_w = h // m1, w // m2
800 + for t in range(1, self.times):
801 + cur_h, cur_w = cur_h // m1, cur_w // m2
802 + hs = hs.reshape(cur_h, m1, cur_w, m2, -1)
803 + hs = hs.permute(0, 2, 1, 3, 4).reshape(
804 + cur_h * cur_w,
805 + m1 * m2 * hs.shape[-1],
806 + )
807 + hs = self.mlp[t](hs)
808 +
809 + results.append(hs)
810 +
811 + return results
812 +
813 +
814 +@MULTIMODAL_REGISTRY.register_processor(
815 + MiniCPMV4_6MultiModalProcessor,
816 + info=MiniCPMV4_6ProcessingInfo,
817 + dummy_inputs=MiniCPMVDummyInputsBuilder,
818 +)
819 +class MiniCPMV4_6ForConditionalGeneration(
820 + nn.Module,
821 + SupportsMultiModal,
822 + SupportsPP,
823 + HasInnerState,
824 + IsHybrid,
825 + SupportsMRoPE,
826 +):
827 + supports_encoder_tp_data = True
828 +
829 + hf_to_vllm_mapper = WeightsMapper(
830 + orig_to_new_prefix={
831 + # transformers v5.7+ uses `vision_tower` and nests `vit_merger`
832 + # inside it. Order matters: more specific prefix must come first.
833 + "model.vision_tower.vit_merger.": "vit_merger.",
834 + "model.vision_tower.": "vpm.",
835 + "model.vpm.": "vpm.",
836 + "model.vit_merger.": "vit_merger.",
837 + "model.merger.": "merger.",
838 + "model.language_model.": "language_model.model.",
839 + "lm_head.": "language_model.lm_head.",
840 + }
841 + )
842 +
843 + packed_modules_mapping = {
844 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],
845 + "gate_up_proj": ["gate_proj", "up_proj"],
846 + "in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],
847 + "in_proj_ba": ["in_proj_b", "in_proj_a"],
848 + }
849 +
850 + @classmethod
851 + def get_placeholder_str(cls, modality: str, i: int) -> str | None:
852 + # transformers v5.7+ chat_template uses these tokens.
853 + if modality.startswith("image"):
854 + return "<|image_pad|>"
855 + if modality.startswith("video"):
856 + return "<|video_pad|>"
857 + raise ValueError("Only image or video modality is supported")
858 +
859 + def get_mrope_input_positions(
860 + self,
861 + input_tokens: list[int],
862 + mm_features: list["MultiModalFeatureSpec"],
863 + ) -> tuple[torch.Tensor, int]:
864 + """MiniCPM-V uses embedding injection for vision, not spatial M-RoPE.
865 +
866 + All tokens (text and vision placeholders) get identical sequential
867 + positions duplicated across the 3 M-RoPE channels expected by the
868 + Qwen3.5 backbone.
869 + """
870 + seq_len = len(input_tokens)
871 + positions = torch.arange(seq_len).unsqueeze(0).expand(3, -1)
872 + return positions, 0
873 +
874 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
875 + super().__init__()
876 + config: MiniCPMV4_6Config = vllm_config.model_config.hf_config
877 + quant_config = vllm_config.quant_config
878 + multimodal_config = vllm_config.model_config.multimodal_config
879 +
880 + self.config = config
881 + self.multimodal_config = multimodal_config
882 + self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
883 +
884 + # --- Vision tower ---
885 + with self._mark_tower_model(vllm_config, {"image"}):
886 + self.vpm = Idefics2VisionTransformer(
887 + config.vision_config,
888 + quant_config=quant_config,
889 + apply_encoder_attention_mask=True,
890 + prefix=maybe_prefix(prefix, "vpm"),
891 + )
892 + if config.drop_vision_last_layer:
893 + self.vpm.encoder.layers = self.vpm.encoder.layers[:-1]
894 +
895 + self.vit_merger = MiniCPMV4_6ViTWindowAttentionMerger(
896 + config.vision_config,
897 + quant_config=quant_config,
898 + prefix=maybe_prefix(prefix, "vit_merger"),
899 + )
900 + self.merger = MiniCPMV4_6Merger(
901 + hidden_size=config.vision_config.hidden_size,
902 + llm_embed_dim=config.text_config.hidden_size,
903 + )
904 +
905 + # --- Language model ---
906 + # Temporarily swap top-level model_type so that Qwen3_5ForCausalLM
907 + # picks up the expected text config when introspecting the hf config.
908 + with self._mark_language_model(vllm_config):
909 + saved_model_type = config.model_type
910 + config.model_type = "qwen3_5_text"
911 + try:
912 + self.language_model = Qwen3_5ForCausalLM(
913 + vllm_config=vllm_config,
914 + prefix=maybe_prefix(prefix, "language_model"),
915 + )
916 + finally:
917 + config.model_type = saved_model_type
918 +
919 + self.make_empty_intermediate_tensors = (
920 + self.language_model.make_empty_intermediate_tensors
921 + )
922 +
923 + # ----- Multimodal parsing -----
924 +
925 + def _parse_and_validate_vision_input(
926 + self,
927 + **kwargs: object,
928 + ) -> MiniCPMVImagePixelInputs | MiniCPMVImageEmbeddingInputs | None:
929 + pixel_values = kwargs.pop("pixel_values", None)
930 + image_embeds = kwargs.pop("image_embeds", None)
931 +
932 + if pixel_values is None and image_embeds is None:
933 + return None
934 +
935 + if image_embeds is not None:
936 + return MiniCPMVImageEmbeddingInputs(
937 + type="image_embeds",
938 + image_embeds=image_embeds,
939 + )
940 +
941 + tgt_sizes = kwargs.pop("tgt_sizes")
942 + num_slices_flat = torch.tensor([len(ps) for ps in pixel_values])
943 + pixel_values_flat = flatten_bn(pixel_values)
944 + tgt_sizes_flat = flatten_bn(tgt_sizes, concat=True)
945 +
946 + return MiniCPMVImagePixelInputs(
947 + type="pixel_values",
948 + pixel_values=pixel_values_flat,
949 + tgt_sizes=tgt_sizes_flat,
950 + num_slices=num_slices_flat,
951 + )
952 +
953 + # ----- Vision forward -----
954 +
955 + def get_vision_hidden_states(
956 + self,
957 + data: MiniCPMVImagePixelInputs,
958 + downsample_mode: str | None = None,
959 + ) -> list[torch.Tensor]:
960 + pixel_values = data["pixel_values"]
961 + tgt_sizes = data["tgt_sizes"]
962 +
963 + B = len(pixel_values)
964 + P = pixel_values[0].shape[-2]
965 + L = max(item.shape[-1] for item in pixel_values)
966 + device = pixel_values[0].device
967 + target_dtype = self.vpm.embeddings.patch_embedding.weight.dtype
968 +
969 + all_pixel_values = torch.zeros(
970 + B,
971 + 3,
972 + P,
973 + L,
974 + dtype=target_dtype,
975 + device=device,
976 + )
977 + for i, pv in enumerate(pixel_values):
978 + all_pixel_values[i, ..., : pv.shape[-1]] = pv.to(target_dtype)
979 +
980 + num_patches = tgt_sizes.prod(-1)
981 + max_patches = int(num_patches.max().item())
982 + patch_attn_mask = torch.zeros(
983 + B,
984 + max_patches,
985 + dtype=torch.bool,
986 + device=device,
987 + )
988 + for i in range(B):
989 + patch_attn_mask[i, : num_patches[i]] = True
990 +
991 + hidden_states = self.vpm.embeddings(
992 + all_pixel_values,
993 + patch_attention_mask=patch_attn_mask.unsqueeze(1),
994 + tgt_sizes=tgt_sizes,
995 + )
996 +
997 + if torch.any(~patch_attn_mask):
998 + mask_dtype = hidden_states.dtype
999 + min_val = torch.finfo(mask_dtype).min
1000 + attention_mask = (~patch_attn_mask).to(dtype=mask_dtype) * min_val
1001 + attention_mask = attention_mask[:, None, None, :]
1002 + else:
1003 + attention_mask = None
1004 +
1005 + # Encoder layers with mid-encoder merger injection
1006 + insert_layer_id = getattr(self.config, "insert_layer_id", -1)
1007 + if downsample_mode is None:
1008 + downsample_mode = getattr(self.config, "downsample_mode", "16x")
1009 + use_vit_merger = downsample_mode != "4x" and insert_layer_id >= 0
1010 +
1011 + for layer in self.vpm.encoder.layers[: insert_layer_id + 1]:
1012 + hidden_states = layer(hidden_states, attention_mask=attention_mask)
1013 +
1014 + if use_vit_merger:
1015 + hidden_states, tgt_sizes, attention_mask = self.vit_merger(
1016 + hidden_states,
1017 + tgt_sizes,
1018 + attention_mask,
1019 + )
1020 +
1021 + for layer in self.vpm.encoder.layers[insert_layer_id + 1 :]:
1022 + hidden_states = layer(hidden_states, attention_mask=attention_mask)
1023 +
1024 + # 4. Post layernorm
1025 + hidden_states = self.vpm.post_layernorm(hidden_states)
1026 +
1027 + # 5. MLP merger → list of per-slice tensors
1028 + return self.merger(hidden_states, tgt_sizes)
1029 +
1030 + def _process_vision_input(self, image_input, use_vit_merger=None):
1031 + if image_input["type"] == "image_embeds":
1032 + return image_input["image_embeds"]
1033 +
1034 + downsample_mode = None
1035 + if use_vit_merger is not None:
1036 + downsample_mode = "16x" if use_vit_merger else "4x"
1037 + image_features = self.get_vision_hidden_states(
1038 + image_input,
1039 + downsample_mode=downsample_mode,
1040 + )
1041 + num_slices = image_input["num_slices"]
1042 + results = []
1043 + idx = 0
1044 + for n in num_slices.tolist():
1045 + group = image_features[idx : idx + n]
1046 + results.append(torch.cat(group, dim=0))
1047 + idx += n
1048 + return results
1049 +
1050 + # ----- Multimodal embedding interface -----
1051 +
1052 + def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
1053 + use_vit_merger_tensors = kwargs.pop("use_vit_merger", None)
1054 + use_vit_merger = None
1055 + if use_vit_merger_tensors is not None:
1056 + if isinstance(use_vit_merger_tensors, torch.Tensor):
1057 + use_vit_merger = bool(use_vit_merger_tensors.any().item())
1058 + elif isinstance(use_vit_merger_tensors, list | tuple):
1059 + use_vit_merger = any(
1060 + bool(t.any().item()) if isinstance(t, torch.Tensor) else bool(t)
1061 + for t in use_vit_merger_tensors
1062 + )
1063 +
1064 + # Split kwargs into image / video buckets (videos are processed via
1065 + # the same vision pipeline; their fields just carry a ``video_`` prefix).
1066 + image_kwargs = {
1067 + k: v
1068 + for k, v in kwargs.items()
1069 + if k in ("pixel_values", "image_embeds", "tgt_sizes")
1070 + }
1071 + video_kwargs = {
1072 + k.removeprefix("video_"): v
1073 + for k, v in kwargs.items()
1074 + if k.startswith("video_")
1075 + }
1076 +
1077 + multimodal_embeddings: tuple[torch.Tensor, ...] = ()
1078 +
1079 + if (
1080 + image_kwargs.get("pixel_values") is not None
1081 + or image_kwargs.get("image_embeds") is not None
1082 + ):
1083 + image_input = self._parse_and_validate_vision_input(**image_kwargs)
1084 + if image_input is not None:
1085 + multimodal_embeddings += tuple(
1086 + self._process_vision_input(
1087 + image_input,
1088 + use_vit_merger=use_vit_merger,
1089 + )
1090 + )
1091 +
1092 + if (
1093 + video_kwargs.get("pixel_values") is not None
1094 + or video_kwargs.get("image_embeds") is not None
1095 + ):
1096 + video_input = self._parse_and_validate_vision_input(**video_kwargs)
1097 + if video_input is not None:
1098 + multimodal_embeddings += tuple(
1099 + self._process_vision_input(
1100 + video_input,
1101 + use_vit_merger=use_vit_merger,
1102 + )
1103 + )
1104 +
1105 + if not multimodal_embeddings:
1106 + return []
1107 + return multimodal_embeddings
1108 +
1109 + def embed_input_ids(
1110 + self,
1111 + input_ids: torch.Tensor,
1112 + multimodal_embeddings: MultiModalEmbeddings | None = None,
1113 + *,
1114 + is_multimodal: torch.Tensor | None = None,
1115 + ) -> torch.Tensor:
1116 + inputs_embeds = self._embed_text_input_ids(
1117 + input_ids,
1118 + self.language_model.embed_input_ids,
1119 + is_multimodal=is_multimodal,
1120 + )
1121 + if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
1122 + return inputs_embeds
1123 +
1124 + is_multimodal = _require_is_multimodal(is_multimodal)
1125 + return _merge_multimodal_embeddings(
1126 + inputs_embeds=inputs_embeds,
1127 + multimodal_embeddings=multimodal_embeddings,
1128 + is_multimodal=is_multimodal,
1129 + )
1130 +
1131 + # ----- Forward / Logits -----
1132 +
1133 + def forward(
1134 + self,
1135 + input_ids: torch.Tensor | None,
1136 + positions: torch.Tensor,
1137 + intermediate_tensors: IntermediateTensors | None = None,
1138 + inputs_embeds: torch.Tensor | None = None,
1139 + **kwargs: Any,
1140 + ) -> torch.Tensor:
1141 + if intermediate_tensors is not None:
1142 + inputs_embeds = None
1143 +
1144 + return self.language_model.model(
1145 + input_ids=input_ids,
1146 + positions=positions,
1147 + intermediate_tensors=intermediate_tensors,
1148 + inputs_embeds=inputs_embeds,
1149 + )
1150 +
1151 + def compute_logits(
1152 + self,
1153 + hidden_states: torch.Tensor,
1154 + ) -> torch.Tensor | None:
1155 + return self.language_model.compute_logits(hidden_states)
1156 +
1157 + # ----- Weight loading -----
1158 +
1159 + def load_weights(
1160 + self,
1161 + weights: Iterable[tuple[str, torch.Tensor]],
1162 + ) -> set[str]:
1163 + loader = AutoWeightsLoader(self, skip_prefixes=["mtp."])
1164 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1165 +
1166 + def get_mm_mapping(self) -> MultiModelKeys:
1167 + return MultiModelKeys.from_string_field(
1168 + language_model="language_model",
1169 + connector=["vit_merger", "merger"],
1170 + tower_model="vpm",
1171 + )
1172 +
1173 + # ----- Mamba / Hybrid state helpers (same as Qwen3.5 VLM) -----
1174 +
1175 + @classmethod
1176 + def get_mamba_state_dtype_from_config(cls, vllm_config):
1177 + return MambaStateDtypeCalculator.gated_delta_net_state_dtype(
1178 + vllm_config.model_config.dtype,
1179 + vllm_config.cache_config.mamba_cache_dtype,
1180 + vllm_config.cache_config.mamba_ssm_cache_dtype,
1181 + )
1182 +
1183 + @classmethod
1184 + def get_mamba_state_shape_from_config(cls, vllm_config):
1185 + parallel_config = vllm_config.parallel_config
1186 + hf_config = vllm_config.model_config.hf_text_config
1187 + tp_size = parallel_config.tensor_parallel_size
1188 + num_spec = (
1189 + vllm_config.speculative_config.num_speculative_tokens
1190 + if vllm_config.speculative_config
1191 + else 0
1192 + )
1193 + return MambaStateShapeCalculator.gated_delta_net_state_shape(
1194 + tp_size,
1195 + hf_config.linear_num_key_heads,
1196 + hf_config.linear_num_value_heads,
1197 + hf_config.linear_key_head_dim,
1198 + hf_config.linear_value_head_dim,
1199 + hf_config.linear_conv_kernel_dim,
1200 + num_spec,
1201 + )
1202 +
1203 + @classmethod
1204 + def get_mamba_state_copy_func(cls):
1205 + return MambaStateCopyFuncCalculator.gated_delta_net_state_copy_func()
vllm/model_executor/models/registry.py
+4
@@ -478,6 +478,10 @@ _MULTIMODAL_MODELS = {
478 ),
479 "MiniCPMO": ("minicpmo", "MiniCPMO"),
480 "MiniCPMV": ("minicpmv", "MiniCPMV"),
481 + "MiniCPMV4_6ForConditionalGeneration": (
482 + "minicpmv4_6",
483 + "MiniCPMV4_6ForConditionalGeneration",
484 + ),
485 "Mistral3ForConditionalGeneration": (
486 "mistral3",
487 "Mistral3ForConditionalGeneration",
vllm/transformers_utils/chat_templates/registry.py
+1
@@ -39,6 +39,7 @@ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = {
39 "deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja",
40 "fuyu": CHAT_TEMPLATES_DIR / "template_fuyu.jinja",
41 "minicpmv": _get_minicpmv_chat_template_fallback,
42 + "minicpmv4_6": _get_minicpmv_chat_template_fallback,
43 "paligemma": CHAT_TEMPLATES_DIR / "template_basic.jinja",
44 "qwen": _get_qwen_chat_template_fallback,
45 "siglip": CHAT_TEMPLATES_DIR / "template_basic.jinja",