[Bugfix][MiniCPM-V] Route video_embeds to the shared vision parser (#54633)
Signed-off-by: subhash polisetti <subhashr161347@gmail.com>
Subhash Polisetti committed
Sep 1, 2026 at 02:16 UTC
4707679cd26459caef158b084f17c41489b4d0fa
3 files changed
+75
-8
tests/model_executor/test_minicpmv.py
new
+56
@@ -0,0 +1,56 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import pytest
4
+import torch
5
+
6
+from vllm.model_executor.models.minicpmv import MiniCPMV2_6
7
+from vllm.model_executor.models.minicpmv4_6 import (
8
+ MiniCPMV4_6ForConditionalGeneration,
9
+)
10
+
11
+pytestmark = pytest.mark.skip_global_cleanup
12
+
13
+
14
+def make_model() -> MiniCPMV2_6:
15
+ return object.__new__(MiniCPMV2_6)
16
+
17
+
18
+def make_model_4_6() -> MiniCPMV4_6ForConditionalGeneration:
19
+ model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
20
+ model._process_vision_input = lambda vision_input, use_vit_merger=None: [
21
+ vision_input["image_embeds"]
22
+ ]
23
+ return model
24
+
25
+
26
+def test_video_embeds_reach_the_vision_parser():
27
+ embeds = torch.arange(32, dtype=torch.float32).reshape(1, 4, 8)
28
+
29
+ modalities = make_model()._parse_and_validate_multimodal_inputs(video_embeds=embeds)
30
+
31
+ video_input = modalities["videos"]
32
+ assert video_input is not None
33
+ assert video_input["type"] == "image_embeds"
34
+ assert torch.equal(video_input["image_embeds"], embeds)
35
+
36
+
37
+def test_video_embeds_are_embedded_by_4_6():
38
+ embeds = torch.arange(32, dtype=torch.float32).reshape(1, 4, 8)
39
+
40
+ embeddings = make_model_4_6().embed_multimodal(video_embeds=embeds)
41
+
42
+ assert len(embeddings) == 1
43
+ assert torch.equal(embeddings[0], embeds)
44
+
45
+
46
+def test_image_and_video_embeds_stay_in_their_own_modality():
47
+ image_embeds = torch.zeros(1, 4, 8)
48
+ video_embeds = torch.ones(1, 4, 8)
49
+
50
+ modalities = make_model()._parse_and_validate_multimodal_inputs(
51
+ image_embeds=image_embeds,
52
+ video_embeds=video_embeds,
53
+ )
54
+
55
+ assert torch.equal(modalities["images"]["image_embeds"], image_embeds)
56
+ assert torch.equal(modalities["videos"]["image_embeds"], video_embeds)
vllm/model_executor/models/minicpmv.py
+17
-1
@@ -469,6 +469,22 @@ def get_version_by_config(config: PretrainedConfig) -> tuple[int, ...]:
469
return tuple(int(x) for x in version_str.split("."))
470
471
472
+_VIDEO_TO_IMAGE_KWARGS = {
473
+ "video_pixel_values": "pixel_values",
474
+ "video_image_sizes": "image_sizes",
475
+ "video_tgt_sizes": "tgt_sizes",
476
+ "video_embeds": "image_embeds",
477
+}
478
+
479
+
480
+def _image_kwargs_from_video(kwargs: Mapping[str, object]) -> dict[str, object]:
481
+ return {
482
+ _VIDEO_TO_IMAGE_KWARGS[k]: v
483
+ for k, v in kwargs.items()
484
+ if k in _VIDEO_TO_IMAGE_KWARGS
485
+ }
486
+
487
+
488
def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):
489
return dict(
490
pixel_values=MultiModalFieldConfig.batched("image"),
@@ -1276,7 +1292,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
1292
and "videos" not in modalities
1293
):
1294
modalities["videos"] = self._parse_and_validate_vision_input(
1279
- "videos", **{k.removeprefix("video_"): v for k, v in kwargs.items()}
1295
+ "videos", **_image_kwargs_from_video(kwargs)
1296
)
1297
1298
return modalities
vllm/model_executor/models/minicpmv4_6.py
+2
-7
@@ -59,6 +59,7 @@ from .minicpmv import (
59
MiniCPMVMultiModalProcessor,
60
MiniCPMVProcessingInfo,
61
MiniCPMVVideoEmbeddingItems,
62
+ _image_kwargs_from_video,
63
)
64
from .module_mapping import MultiModelKeys
65
from .qwen3_5 import Qwen3_5ForCausalLM
@@ -1200,18 +1201,12 @@ class MiniCPMV4_6ForConditionalGeneration(
1201
for t in use_vit_merger_tensors
1202
)
1203
1203
- # Split kwargs into image / video buckets (videos are processed via
1204
- # the same vision pipeline; their fields just carry a ``video_`` prefix).
1204
image_kwargs = {
1205
k: v
1206
for k, v in kwargs.items()
1207
if k in ("pixel_values", "image_embeds", "tgt_sizes")
1208
}
1210
- video_kwargs = {
1211
- k.removeprefix("video_"): v
1212
- for k, v in kwargs.items()
1213
- if k.startswith("video_")
1214
- }
1209
+ video_kwargs = _image_kwargs_from_video(kwargs)
1210
1211
multimodal_embeddings: tuple[torch.Tensor, ...] = ()
1212