A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 1, 22:30 UTC
[Bugfix] MiniCPM-V-4.6 video inference crash: placeholder count mismatches visual embedding count (#44509)
Signed-off-by: tc-mb <tianchi_cai@icloud.com>
tc-mb committed
Jun 4, 2026 at 23:22 UTC
3dbb4e0acef59930bef53b5b8e15ed869cf0a499
2 files changed
+60
-2
vllm/model_executor/models/minicpmv4_6.py
+52
-1
@@ -5,7 +5,9 @@
5
from collections.abc import Iterable, Mapping
6
from typing import Any
7
8
+import numpy as np
9
import torch
10
+from PIL import Image as PILImage
11
from torch import nn
12
from transformers import MiniCPMV4_6Config
13
@@ -30,7 +32,7 @@ from vllm.multimodal.inputs import (
32
MultiModalFieldConfig,
33
NestedTensors,
34
)
33
-from vllm.multimodal.parse import ImageProcessorItems, VideoProcessorItems
35
+from vllm.multimodal.parse import ImageProcessorItems, ImageSize, VideoProcessorItems
36
from vllm.multimodal.processing.processor import (
37
PromptReplacement,
38
PromptUpdateDetails,
@@ -239,12 +241,34 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
241
242
per_video_pixel_values: list[torch.Tensor] = []
243
per_video_tgt_sizes: list[torch.Tensor] = []
244
+ per_video_image_sizes: list[torch.Tensor] = []
245
246
for video in parsed_videos:
247
# video is iterable of frames (PIL Image or numpy array).
248
all_slices: list[torch.Tensor] = []
249
ts_list: list[torch.Tensor] = []
250
+ frame_sizes: list[torch.Tensor] = []
251
for frame in video:
252
+ # Record per-frame (W, H) for video_image_sizes so that
253
+ # get_video_prompt_texts can consume a consistent frame size.
254
+ if isinstance(frame, PILImage.Image):
255
+ w, h = frame.size
256
+ elif isinstance(frame, np.ndarray):
257
+ if frame.ndim == 3 and frame.shape[-1] in (1, 3, 4):
258
+ # HWC (e.g. from np.array(PIL.Image))
259
+ h, w = frame.shape[0], frame.shape[1]
260
+ else:
261
+ # CHW
262
+ _, h, w = frame.shape
263
+ elif isinstance(frame, torch.Tensor):
264
+ if frame.ndim == 3 and frame.shape[-1] in (1, 3, 4):
265
+ h, w = frame.shape[0], frame.shape[1]
266
+ else:
267
+ _, h, w = frame.shape
268
+ else:
269
+ raise TypeError(f"Unsupported frame type: {type(frame)}")
270
+ frame_sizes.append(torch.tensor([w, h], dtype=torch.long, device="cpu"))
271
+
272
ip_out = image_processor([frame], **video_mm_kwargs)
273
pv = ip_out["pixel_values"] # (1, C, P, sum_W)
274
ts = ip_out["target_sizes"] # (n_slices, 2)
@@ -275,6 +299,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
299
300
per_video_pixel_values.append(out)
301
per_video_tgt_sizes.append(torch.cat(ts_list, dim=0))
302
+ per_video_image_sizes.append(torch.stack(frame_sizes))
303
304
if not per_video_pixel_values:
305
return {}
@@ -282,6 +307,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
307
return {
308
"video_pixel_values": per_video_pixel_values,
309
"video_tgt_sizes": per_video_tgt_sizes,
310
+ "video_image_sizes": per_video_image_sizes,
311
}
312
313
def _get_prompt_updates(
@@ -327,6 +353,31 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
353
)
354
355
def get_video_replacement(item_idx: int):
356
+ # Prefer video_image_sizes from processed data so that the
357
+ # placeholder count is driven by the same frame sizes that the
358
+ # vision tower will actually consume.
359
+ video_mm_kwargs = out_mm_kwargs.get("video")
360
+ if video_mm_kwargs is not None and item_idx < len(video_mm_kwargs):
361
+ video_item = video_mm_kwargs[item_idx]
362
+ image_sizes_elem = video_item.get("video_image_sizes")
363
+ if image_sizes_elem is not None and image_sizes_elem.data is not None:
364
+ # image_sizes_elem.data: (num_frames, 2) – each row is [W, H]
365
+ image_sizes = image_sizes_elem.data
366
+ num_frames = image_sizes.shape[0]
367
+ frame_size = ImageSize(
368
+ width=int(image_sizes[0, 0].item()),
369
+ height=int(image_sizes[0, 1].item()),
370
+ )
371
+ return PromptUpdateDetails.select_text(
372
+ self.get_video_prompt_texts(
373
+ frame_size,
374
+ num_frames,
375
+ downsample_mode=ds_mode,
376
+ video_idx=item_idx,
377
+ ),
378
+ video_embed_text,
379
+ )
380
+
381
videos = mm_items.get_items(
382
"video",
383
(MiniCPMVVideoEmbeddingItems, VideoProcessorItems),
vllm/multimodal/parse.py
+8
-1
@@ -378,7 +378,14 @@ class VideoProcessorItems(ProcessorBatchItems[HfVideoItem | None]):
378
if isinstance(image, PILImage.Image):
379
return ImageSize(*image.size)
380
if isinstance(image, (np.ndarray, torch.Tensor)):
381
- _, h, w = image.shape
381
+ if image.ndim == 3 and image.shape[-1] in (1, 3, 4):
382
+ # HWC format (e.g. from np.array(PIL.Image) via
383
+ # _get_video_with_metadata). PIL images are always
384
+ # channels-last.
385
+ h, w = image.shape[0], image.shape[1]
386
+ else:
387
+ # CHW format (standard PyTorch / numpy convention).
388
+ _, h, w = image.shape
389
return ImageSize(w, h)
390
391
assert_never(image)