| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | # Adapted from |
| 5 | # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py |
| 6 | # Copyright 2023 The vLLM team. |
| 7 | # Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. |
| 8 | # |
| 9 | # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX |
| 10 | # and OPT implementations in this library. It has been modified from its |
| 11 | # original forms to accommodate minor architectural differences compared |
| 12 | # to GPT-NeoX and OPT used by the Meta AI team that trained the model. |
| 13 | # |
| 14 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 15 | # you may not use this file except in compliance with the License. |
| 16 | # You may obtain a copy of the License at |
| 17 | # |
| 18 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 19 | # |
| 20 | # Unless required by applicable law or agreed to in writing, software |
| 21 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 22 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 23 | # See the License for the specific language governing permissions and |
| 24 | # limitations under the License. |
| 25 | """Inference-only MiniCPM-V model compatible with HuggingFace weights.""" |
| 26 | |
| 27 | import math |
| 28 | from collections import defaultdict |
| 29 | from collections.abc import Callable, Hashable, Iterable, Mapping, Sequence, Set |
| 30 | from functools import cached_property, partial |
| 31 | from itertools import chain |
| 32 | from typing import Annotated, Any, ClassVar, Literal, TypeAlias, cast |
| 33 | |
| 34 | import numpy as np |
| 35 | import torch |
| 36 | import torch.types |
| 37 | from torch import nn |
| 38 | from torch.nn.init import trunc_normal_ |
| 39 | from transformers import BatchFeature, PreTrainedConfig |
| 40 | from transformers.dynamic_module_utils import ( |
| 41 | get_class_from_dynamic_module, |
| 42 | resolve_trust_remote_code, |
| 43 | ) |
| 44 | from typing_extensions import TypedDict, TypeVar |
| 45 | |
| 46 | from vllm.config import VllmConfig |
| 47 | from vllm.config.multimodal import ( |
| 48 | ImageDummyOptions, |
| 49 | MultiModalDummyOptions, |
| 50 | ) |
| 51 | from vllm.inputs import ModalityData, MultiModalDataDict |
| 52 | from vllm.model_executor.layers.quantization import QuantizationConfig |
| 53 | from vllm.model_executor.layers.resampler import ( |
| 54 | BaseResampler, |
| 55 | Resampler2, |
| 56 | get_2d_sincos_pos_embed, |
| 57 | ) |
| 58 | from vllm.model_executor.models.llama import LlamaForCausalLM |
| 59 | from vllm.model_executor.models.minicpm import MiniCPMForCausalLM |
| 60 | from vllm.model_executor.models.module_mapping import MultiModelKeys |
| 61 | from vllm.model_executor.models.qwen2 import Qwen2ForCausalLM |
| 62 | from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM |
| 63 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 64 | from vllm.multimodal.inputs import ( |
| 65 | MultiModalFieldConfig, |
| 66 | MultiModalKwargsItems, |
| 67 | NestedTensors, |
| 68 | ) |
| 69 | from vllm.multimodal.parse import ( |
| 70 | DictEmbeddingItems, |
| 71 | ImageItem, |
| 72 | ImageProcessorItems, |
| 73 | ImageSize, |
| 74 | ModalityDataItems, |
| 75 | MultiModalDataItems, |
| 76 | MultiModalDataParser, |
| 77 | VideoItem, |
| 78 | VideoProcessorItems, |
| 79 | ) |
| 80 | from vllm.multimodal.processing import BaseDummyInputsBuilder |
| 81 | from vllm.multimodal.processing.processor import ( |
| 82 | BaseMultiModalProcessor, |
| 83 | BaseProcessingInfo, |
| 84 | MultiModalPromptUpdates, |
| 85 | PlaceholderFeaturesInfo, |
| 86 | PromptReplacement, |
| 87 | PromptUpdate, |
| 88 | PromptUpdateDetails, |
| 89 | ResolvedPromptUpdate, |
| 90 | cached_encode, |
| 91 | ) |
| 92 | from vllm.platforms import current_platform |
| 93 | from vllm.sequence import IntermediateTensors |
| 94 | from vllm.transformers_utils.processor import ( |
| 95 | _merge_mm_kwargs, |
| 96 | cached_get_image_processor, |
| 97 | ) |
| 98 | from vllm.transformers_utils.utils import convert_model_repo_to_path |
| 99 | from vllm.utils.collection_utils import flatten_2d_lists, is_list_of |
| 100 | from vllm.utils.gpu_sync_debug import gpu_sync_allowed |
| 101 | from vllm.utils.tensor_schema import TensorSchema, TensorShape |
| 102 | from vllm.utils.torch_utils import async_tensor_h2d, set_default_torch_dtype |
| 103 | from vllm.v1.worker.encoder_cudagraph_defs import ( |
| 104 | ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG, |
| 105 | EncoderCudaGraphCaptureInputs, |
| 106 | EncoderCudaGraphConfig, |
| 107 | EncoderCudaGraphReplayBuffers, |
| 108 | EncoderItemSpec, |
| 109 | ) |
| 110 | |
| 111 | from .idefics2_vision_model import Idefics2VisionTransformer |
| 112 | from .interfaces import ( |
| 113 | MultiModalEmbeddings, |
| 114 | SupportsEncoderCudaGraph, |
| 115 | SupportsLoRA, |
| 116 | SupportsMultiModal, |
| 117 | SupportsPP, |
| 118 | ) |
| 119 | from .utils import AutoWeightsLoader, WeightsMapper, flatten_bn, maybe_prefix |
| 120 | |
| 121 | # For profile run |
| 122 | _MAX_FRAMES_PER_VIDEO = 16 |
| 123 | |
| 124 | |
| 125 | class MiniCPMVImagePixelInputs(TensorSchema): |
| 126 | """Dimensions: |
| 127 | - bns: Batch size * number of images * number of slices |
| 128 | - bn: Batch size * number of images |
| 129 | - c: Number of channels |
| 130 | - h: Height |
| 131 | - w: Width |
| 132 | """ |
| 133 | |
| 134 | type: Literal["pixel_values"] = "pixel_values" |
| 135 | |
| 136 | # Note that the patch size may vary, so we pass it as a list instead of a |
| 137 | # batched tensor. |
| 138 | pixel_values: Annotated[ |
| 139 | list[torch.Tensor], |
| 140 | TensorShape("bns", "c", "h", "w", dynamic_dims={"h", "w"}), |
| 141 | ] |
| 142 | tgt_sizes: Annotated[ |
| 143 | torch.Tensor, |
| 144 | TensorShape("bns", 2), # This should be in `(height, width)` format. |
| 145 | ] |
| 146 | num_slices: Annotated[ |
| 147 | torch.Tensor, |
| 148 | TensorShape("bn"), |
| 149 | ] |
| 150 | |
| 151 | |
| 152 | class MiniCPMVImageEmbeddingInputs(TensorSchema): |
| 153 | """Dimensions: |
| 154 | - bn: Batch size * number of images |
| 155 | - ns: Number of slices |
| 156 | - hs: Hidden size (must match language model backbone) |
| 157 | """ |
| 158 | |
| 159 | type: Literal["image_embeds"] |
| 160 | image_embeds: Annotated[ |
| 161 | torch.Tensor | list[torch.Tensor], |
| 162 | TensorShape("bn", "ns", "hs", dynamic_dims={"ns"}), |
| 163 | ] |
| 164 | |
| 165 | |
| 166 | MiniCPMVImageInputs: TypeAlias = MiniCPMVImagePixelInputs | MiniCPMVImageEmbeddingInputs |
| 167 | |
| 168 | DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6) |
| 169 | |
| 170 | |
| 171 | class Resampler2_5(BaseResampler): |
| 172 | def __init__( |
| 173 | self, |
| 174 | num_queries: int, |
| 175 | embed_dim: int, |
| 176 | num_heads: int, |
| 177 | kv_dim: int | None = None, |
| 178 | norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN, |
| 179 | max_size: tuple[int, int] = (70, 70), |
| 180 | quant_config: QuantizationConfig | None = None, |
| 181 | prefix: str = "", |
| 182 | ) -> None: |
| 183 | super().__init__( |
| 184 | num_queries, |
| 185 | embed_dim, |
| 186 | num_heads, |
| 187 | kv_dim, |
| 188 | norm_layer, |
| 189 | quant_config=quant_config, |
| 190 | prefix=prefix, |
| 191 | ) |
| 192 | |
| 193 | self.max_size = max_size |
| 194 | self._set_2d_pos_cache(self.max_size) |
| 195 | |
| 196 | def _set_2d_pos_cache( |
| 197 | self, max_size: tuple[int, int], device: torch.types.Device = "cpu" |
| 198 | ) -> None: |
| 199 | pos_embed_arr = get_2d_sincos_pos_embed( |
| 200 | self.embed_dim, max_size, version=(2, 5) |
| 201 | ) |
| 202 | pos_embed = torch.from_numpy(pos_embed_arr).float().to(device) |
| 203 | self.register_buffer("pos_embed", pos_embed, persistent=False) |
| 204 | |
| 205 | def _adjust_pos_cache( |
| 206 | self, tgt_sizes: torch.Tensor, device: torch.types.Device |
| 207 | ) -> None: |
| 208 | max_h = tgt_sizes[:, 0].max().item() |
| 209 | max_w = tgt_sizes[:, 1].max().item() |
| 210 | assert isinstance(max_h, int) and isinstance(max_w, int) |
| 211 | |
| 212 | if max_h > self.max_size[0] or max_w > self.max_size[1]: |
| 213 | self.max_size = ( |
| 214 | max(max_h, self.max_size[0]), |
| 215 | max(max_w, self.max_size[1]), |
| 216 | ) |
| 217 | self._set_2d_pos_cache(self.max_size, device) |
| 218 | |
| 219 | def forward(self, x: torch.Tensor, tgt_sizes: torch.Tensor) -> torch.Tensor: |
| 220 | assert x.shape[0] == tgt_sizes.shape[0] |
| 221 | bs = x.shape[0] |
| 222 | |
| 223 | device = x.device |
| 224 | dtype = x.dtype |
| 225 | |
| 226 | # Only adjust the pos cache in eager mode; during CUDA graph capture |
| 227 | # the cache was already expanded by the preceding warmup run |
| 228 | if not torch.cuda.is_current_stream_capturing(): |
| 229 | self._adjust_pos_cache(tgt_sizes, device=device) |
| 230 | |
| 231 | # tgt_sizes is pinned-copied only when it comes from the CPU-side |
| 232 | # mm_kwargs; during capture/replay it is already a GPU buffer. |
| 233 | if tgt_sizes.device != x.device: |
| 234 | tgt_sizes = async_tensor_h2d(tgt_sizes, x.device) |
| 235 | patch_len = tgt_sizes[:, 0] * tgt_sizes[:, 1] |
| 236 | max_patch_len = x.shape[1] |
| 237 | |
| 238 | seq_idx = torch.arange(max_patch_len, device=device).unsqueeze(0) |
| 239 | |
| 240 | tgt_w = tgt_sizes[:, 1].unsqueeze(1) |
| 241 | tgt_w = torch.clamp(tgt_w, min=1) |
| 242 | |
| 243 | h_idx = seq_idx // tgt_w |
| 244 | w_idx = seq_idx % tgt_w |
| 245 | |
| 246 | h_idx = torch.clamp(h_idx, max=self.pos_embed.shape[0] - 1) |
| 247 | w_idx = torch.clamp(w_idx, max=self.pos_embed.shape[1] - 1) |
| 248 | |
| 249 | key_padding_mask = seq_idx >= patch_len.unsqueeze(1) |
| 250 | |
| 251 | pos_embed = self.pos_embed[h_idx, w_idx].to(dtype) |
| 252 | pos_embed = torch.where( |
| 253 | key_padding_mask.unsqueeze(-1), torch.zeros_like(pos_embed), pos_embed |
| 254 | ).permute(1, 0, 2) # BLD => L * B * D |
| 255 | |
| 256 | x, _ = self.kv_proj(x) # B * L * D |
| 257 | x = self.ln_kv(x).permute(1, 0, 2) # L * B * D |
| 258 | |
| 259 | q = self.ln_q(self.query) # Q * D |
| 260 | |
| 261 | out = self.attn( |
| 262 | self._repeat(q, bs), # Q * B * D |
| 263 | x + pos_embed, # L * B * D + L * B * D |
| 264 | x, |
| 265 | key_padding_mask=key_padding_mask, |
| 266 | )[0] |
| 267 | # out: Q * B * D |
| 268 | x = out.permute(1, 0, 2) # B * Q * D |
| 269 | |
| 270 | x = self.ln_post(x) |
| 271 | x = x @ self.proj |
| 272 | return x |
| 273 | |
| 274 | |
| 275 | class Resampler4_5(Resampler2_5): |
| 276 | def __init__( |
| 277 | self, |
| 278 | num_queries: int, |
| 279 | embed_dim: int, |
| 280 | num_heads: int, |
| 281 | kv_dim: int | None = None, |
| 282 | norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN, |
| 283 | max_size: tuple[int, int] = (70, 70), |
| 284 | max_temporal_size: int = 36000, |
| 285 | quant_config: QuantizationConfig | None = None, |
| 286 | prefix: str = "", |
| 287 | ) -> None: |
| 288 | super().__init__( |
| 289 | num_queries, |
| 290 | embed_dim, |
| 291 | num_heads, |
| 292 | kv_dim, |
| 293 | norm_layer, |
| 294 | max_size, |
| 295 | quant_config=quant_config, |
| 296 | prefix=prefix, |
| 297 | ) |
| 298 | |
| 299 | trunc_normal_(self.query, std=0.02) |
| 300 | self.max_temporal_size = max_temporal_size |
| 301 | self._set_temporal_pos_cache(self.max_temporal_size) |
| 302 | self.apply(self._init_weights) |
| 303 | |
| 304 | def get_1d_sincos_pos_embed_from_temporal_size( |
| 305 | self, embed_dim: int, pos: np.ndarray |
| 306 | ): |
| 307 | """embed_dim: output dimension for each position |
| 308 | pos: a list of positions to be encoded: size (M,) |
| 309 | out: (M, D) |
| 310 | """ |
| 311 | assert embed_dim % 2 == 0 |
| 312 | omega = np.arange(embed_dim // 2, dtype=np.float32) |
| 313 | omega /= embed_dim / 2.0 |
| 314 | omega = 1.0 / 10000**omega # (D/2,) |
| 315 | |
| 316 | pos = pos.reshape(-1) # (M,) |
| 317 | out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product |
| 318 | |
| 319 | emb_sin = np.sin(out) # (M, D/2) |
| 320 | emb_cos = np.cos(out) # (M, D/2) |
| 321 | |
| 322 | emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) |
| 323 | return emb |
| 324 | |
| 325 | def _set_temporal_pos_cache( |
| 326 | self, max_temporal_size: int, device: torch.types.Device = "cpu" |
| 327 | ) -> None: |
| 328 | temporal_size = np.arange(max_temporal_size, dtype=np.float32) |
| 329 | pos_embed = ( |
| 330 | torch.from_numpy( |
| 331 | self.get_1d_sincos_pos_embed_from_temporal_size( |
| 332 | self.embed_dim, temporal_size |
| 333 | ) |
| 334 | ) |
| 335 | .float() |
| 336 | .to(device) |
| 337 | ) |
| 338 | self.register_buffer("temporal_pos_embed", pos_embed, persistent=False) |
| 339 | |
| 340 | def _adjust_temporal_pos_cache( |
| 341 | self, max_temporal_size: int, device: torch.types.Device = "cpu" |
| 342 | ): |
| 343 | if max_temporal_size > self.max_temporal_size: |
| 344 | self.max_temporal_size = max_temporal_size |
| 345 | self._set_temporal_pos_cache(self.max_temporal_size, device) |
| 346 | |
| 347 | def _init_weights(self, m: nn.Linear | nn.LayerNorm): |
| 348 | if isinstance(m, nn.Linear): |
| 349 | trunc_normal_(m.weight, std=0.02) |
| 350 | if isinstance(m, nn.Linear) and m.bias is not None: |
| 351 | nn.init.constant_(m.bias, 0) |
| 352 | elif isinstance(m, nn.LayerNorm): |
| 353 | nn.init.constant_(m.bias, 0) |
| 354 | nn.init.constant_(m.weight, 1.0) |
| 355 | |
| 356 | def forward( |
| 357 | self, |
| 358 | x: torch.Tensor, |
| 359 | tgt_sizes: torch.Tensor, |
| 360 | # temporal_ids for high refresh rate videos |
| 361 | temporal_ids=None, |
| 362 | ) -> torch.Tensor: |
| 363 | assert x.shape[0] == tgt_sizes.shape[0] |
| 364 | bs = x.shape[0] |
| 365 | |
| 366 | device = x.device |
| 367 | dtype = x.dtype |
| 368 | |
| 369 | patch_len = tgt_sizes[:, 0] * tgt_sizes[:, 1] |
| 370 | |
| 371 | self._adjust_pos_cache(tgt_sizes, device=device) |
| 372 | |
| 373 | temporal_pos_emb = False |
| 374 | temporal_ids_flatten = None |
| 375 | if temporal_ids is not None: |
| 376 | # example: [[-1], [-1], [2, 6, 9]] |
| 377 | temporal_ids_flatten = list(chain.from_iterable(temporal_ids)) |
| 378 | max_temporal_size = max(temporal_ids_flatten, default=0) |
| 379 | if max_temporal_size > -1: |
| 380 | temporal_pos_emb = True |
| 381 | if max_temporal_size > self.max_temporal_size: |
| 382 | self._adjust_temporal_pos_cache(max_temporal_size, device) |
| 383 | |
| 384 | max_patch_len = patch_len.max().item() |
| 385 | assert isinstance(max_patch_len, int) |
| 386 | |
| 387 | key_padding_mask = torch.zeros( |
| 388 | (bs, max_patch_len), dtype=torch.bool, device=device |
| 389 | ) |
| 390 | |
| 391 | x, _ = self.kv_proj(x) # B * L * D |
| 392 | x = self.ln_kv(x).permute(1, 0, 2) # L * B * D |
| 393 | q = self.ln_q(self.query) # Q * D |
| 394 | |
| 395 | pos_embed_2d = [] |
| 396 | pos_embed_temporal = [] |
| 397 | for i in range(bs): |
| 398 | tgt_h, tgt_w = tgt_sizes[i] |
| 399 | if temporal_pos_emb: |
| 400 | assert temporal_ids_flatten is not None |
| 401 | if temporal_ids_flatten[i] == -1: |
| 402 | pos_embed_temporal.append( |
| 403 | torch.zeros(self.embed_dim, dtype=dtype, device=device) |
| 404 | ) |
| 405 | else: |
| 406 | pos_embed_temporal.append( |
| 407 | self.temporal_pos_embed[temporal_ids_flatten[i]].to(dtype) |
| 408 | ) # D |
| 409 | |
| 410 | pos_embed_2d.append( |
| 411 | self.pos_embed[:tgt_h, :tgt_w, :] |
| 412 | .reshape((tgt_h * tgt_w, -1)) |
| 413 | .to(device=device, dtype=dtype) |
| 414 | ) # patches * D |
| 415 | key_padding_mask[i, patch_len[i] :] = True |
| 416 | |
| 417 | pos_embed_2d = torch.nn.utils.rnn.pad_sequence( |
| 418 | pos_embed_2d, batch_first=True, padding_value=0.0 |
| 419 | ).permute(1, 0, 2) # BLD => L * B * D |
| 420 | |
| 421 | k = x + pos_embed_2d |
| 422 | v = x |
| 423 | if pos_embed_temporal: |
| 424 | k += torch.stack(pos_embed_temporal, dim=0) |
| 425 | bs = len(temporal_ids) |
| 426 | merge_k = [] |
| 427 | merge_v = [] |
| 428 | merge_key_padding_mask = [] |
| 429 | |
| 430 | start = 0 |
| 431 | for tp in temporal_ids: |
| 432 | end = start + len(tp) |
| 433 | # L * (end-start) * D -> (end-start) * L * D |
| 434 | # -> 1 * L*(end-start) * D |
| 435 | merge_k.append( |
| 436 | k[:, start:end, :].permute(1, 0, 2).reshape(-1, self.embed_dim) |
| 437 | ) |
| 438 | merge_v.append( |
| 439 | v[:, start:end, :].permute(1, 0, 2).reshape(-1, self.embed_dim) |
| 440 | ) |
| 441 | merge_key_padding_mask.append( |
| 442 | key_padding_mask[start:end, :].reshape(-1, 1) |
| 443 | ) |
| 444 | |
| 445 | start = end |
| 446 | |
| 447 | k = torch.nn.utils.rnn.pad_sequence( |
| 448 | merge_k, batch_first=True, padding_value=0.0 |
| 449 | ).permute(1, 0, 2) # L*(end-start) |
| 450 | v = torch.nn.utils.rnn.pad_sequence( |
| 451 | merge_v, batch_first=True, padding_value=0.0 |
| 452 | ).permute(1, 0, 2) # L*(end-start) |
| 453 | key_padding_mask = torch.nn.utils.rnn.pad_sequence( |
| 454 | merge_key_padding_mask, batch_first=True, padding_value=True |
| 455 | ).squeeze(-1) |
| 456 | |
| 457 | out = self.attn( |
| 458 | self._repeat(q, bs), # Q * B * D |
| 459 | k, # L * B * D + L * B * D |
| 460 | v, |
| 461 | key_padding_mask=key_padding_mask, |
| 462 | )[0] |
| 463 | # out: Q * B * D |
| 464 | x = out.permute(1, 0, 2) # B * Q * D |
| 465 | |
| 466 | x = self.ln_post(x) |
| 467 | x = x @ self.proj |
| 468 | return x |
| 469 | |
| 470 | |
| 471 | def get_version_by_config(config: PreTrainedConfig) -> tuple[int, ...]: |
| 472 | version_float = getattr(config, "version", None) |
| 473 | |
| 474 | # The old configs do not include version number |
| 475 | # TODO: Remove this after the HF repos are updated |
| 476 | if version_float is None: |
| 477 | if config.hidden_size == 2304 and config.query_num == 64: |
| 478 | return (2, 0) |
| 479 | return (2, 5) |
| 480 | version_str = str(version_float) |
| 481 | return tuple(int(x) for x in version_str.split(".")) |
| 482 | |
| 483 | |
| 484 | _VIDEO_TO_IMAGE_KWARGS = { |
| 485 | "video_pixel_values": "pixel_values", |
| 486 | "video_image_sizes": "image_sizes", |
| 487 | "video_tgt_sizes": "tgt_sizes", |
| 488 | "video_embeds": "image_embeds", |
| 489 | } |
| 490 | |
| 491 | |
| 492 | def _image_kwargs_from_video(kwargs: Mapping[str, object]) -> dict[str, object]: |
| 493 | return { |
| 494 | _VIDEO_TO_IMAGE_KWARGS[k]: v |
| 495 | for k, v in kwargs.items() |
| 496 | if k in _VIDEO_TO_IMAGE_KWARGS |
| 497 | } |
| 498 | |
| 499 | |
| 500 | def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]): |
| 501 | return dict( |
| 502 | pixel_values=MultiModalFieldConfig.batched("image"), |
| 503 | image_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True), |
| 504 | tgt_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True), |
| 505 | image_embeds=MultiModalFieldConfig.batched("image"), |
| 506 | video_pixel_values=MultiModalFieldConfig.batched("video"), |
| 507 | video_image_sizes=MultiModalFieldConfig.batched("video", keep_on_cpu=True), |
| 508 | video_tgt_sizes=MultiModalFieldConfig.batched("video", keep_on_cpu=True), |
| 509 | video_embeds=MultiModalFieldConfig.batched("video"), |
| 510 | ) |
| 511 | |
| 512 | |
| 513 | class MiniCPMVImageEmbeddingItems(DictEmbeddingItems): |
| 514 | def __init__( |
| 515 | self, |
| 516 | data: Mapping[str, torch.Tensor], |
| 517 | fields_factory: Callable[ |
| 518 | [Mapping[str, torch.Tensor]], |
| 519 | Mapping[str, MultiModalFieldConfig], |
| 520 | ], |
| 521 | required_fields: Set[str], |
| 522 | optional_fields: Set[str] = frozenset(), |
| 523 | ) -> None: |
| 524 | super().__init__( |
| 525 | data, |
| 526 | modality="image", |
| 527 | required_fields=required_fields, |
| 528 | optional_fields=optional_fields, |
| 529 | fields_factory=fields_factory, |
| 530 | ) |
| 531 | |
| 532 | def get_image_size(self, index: int) -> ImageSize: |
| 533 | image_size = self.get(index)["image_sizes"].tolist() |
| 534 | return ImageSize(width=image_size[0], height=image_size[1]) |
| 535 | |
| 536 | |
| 537 | class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems): |
| 538 | def __init__( |
| 539 | self, |
| 540 | data: Mapping[str, torch.Tensor], |
| 541 | fields_factory: Callable[ |
| 542 | [Mapping[str, torch.Tensor]], |
| 543 | Mapping[str, MultiModalFieldConfig], |
| 544 | ], |
| 545 | required_fields: Set[str], |
| 546 | optional_fields: Set[str] = frozenset(), |
| 547 | ) -> None: |
| 548 | super().__init__( |
| 549 | data, |
| 550 | modality="video", |
| 551 | required_fields=required_fields, |
| 552 | optional_fields=optional_fields, |
| 553 | fields_factory=fields_factory, |
| 554 | ) |
| 555 | |
| 556 | def get_frame_size(self, index: int) -> ImageSize: |
| 557 | frame_size = self.get(index)["video_image_sizes"].tolist() |
| 558 | return ImageSize(width=frame_size[0], height=frame_size[1]) |
| 559 | |
| 560 | def get_num_frames(self, index: int) -> int: |
| 561 | return len(self.get(index)["video_image_sizes"]) |
| 562 | |
| 563 | |
| 564 | class MiniCPMVMultiModalInputs(TypedDict, total=False): |
| 565 | images: MiniCPMVImageInputs | None |
| 566 | videos: MiniCPMVImageInputs | None |
| 567 | |
| 568 | |
| 569 | class MiniCPMVMultiModalDataParser(MultiModalDataParser): |
| 570 | # The per-item sizes are what size the placeholder range. |
| 571 | embedding_fields = { |
| 572 | "image": {"image_embeds": "values", "image_sizes": "metadata"}, |
| 573 | "video": {"video_embeds": "values", "video_image_sizes": "metadata"}, |
| 574 | } |
| 575 | |
| 576 | def _parse_image_data( |
| 577 | self, |
| 578 | data: dict[str, torch.Tensor] | ModalityData[ImageItem], |
| 579 | ) -> ModalityDataItems[Any, Any] | None: |
| 580 | if isinstance(data, dict): |
| 581 | required, optional = self.embedding_field_sets("image") |
| 582 | return MiniCPMVImageEmbeddingItems( |
| 583 | data, |
| 584 | fields_factory=_minicpmv_field_config, |
| 585 | required_fields=required, |
| 586 | optional_fields=optional, |
| 587 | ) |
| 588 | |
| 589 | return super()._parse_image_data(data) |
| 590 | |
| 591 | def _parse_video_data( |
| 592 | self, |
| 593 | data: dict[str, torch.Tensor] | ModalityData[VideoItem], |
| 594 | ) -> ModalityDataItems[Any, Any] | None: |
| 595 | if isinstance(data, dict): |
| 596 | required, optional = self.embedding_field_sets("video") |
| 597 | return MiniCPMVVideoEmbeddingItems( |
| 598 | data, |
| 599 | fields_factory=_minicpmv_field_config, |
| 600 | required_fields=required, |
| 601 | optional_fields=optional, |
| 602 | ) |
| 603 | |
| 604 | return super()._parse_video_data(data) |
| 605 | |
| 606 | |
| 607 | class MiniCPMVProcessingInfo(BaseProcessingInfo): |
| 608 | image_pattern = "(<image>./</image>)" |
| 609 | video_pattern = "(<video>./</video>)" |
| 610 | |
| 611 | @cached_property |
| 612 | def _image_processor_cls(self): |
| 613 | model_config = self.ctx.model_config |
| 614 | model_path = convert_model_repo_to_path(model_config.model) |
| 615 | |
| 616 | from transformers import ImageProcessingMixin |
| 617 | |
| 618 | image_processor_config, _ = ImageProcessingMixin.get_image_processor_dict( |
| 619 | model_path, |
| 620 | revision=model_config.revision, |
| 621 | token=model_config.hf_token, |
| 622 | ) |
| 623 | |
| 624 | auto_map = image_processor_config.get("auto_map") or {} |
| 625 | class_ref = auto_map.get("AutoImageProcessor") |
| 626 | if not class_ref: |
| 627 | raise ValueError( |
| 628 | "Missing auto_map['AutoImageProcessor'] in image processor config " |
| 629 | f"for {model_config.model!r}" |
| 630 | ) |
| 631 | |
| 632 | resolve_trust_remote_code( |
| 633 | model_config.trust_remote_code, |
| 634 | model_config.model, |
| 635 | has_local_code=False, |
| 636 | has_remote_code=True, |
| 637 | ) |
| 638 | return get_class_from_dynamic_module( |
| 639 | class_ref, |
| 640 | model_config.model, |
| 641 | revision=model_config.revision, |
| 642 | ) |
| 643 | |
| 644 | def get_hf_config(self): |
| 645 | return self.ctx.get_hf_config() |
| 646 | |
| 647 | def _get_checkpoint_image_processor(self, **kwargs: object): |
| 648 | model_config = self.ctx.model_config |
| 649 | processor_cls = self._image_processor_cls |
| 650 | merged_kwargs = _merge_mm_kwargs(model_config, processor_cls, **kwargs) |
| 651 | |
| 652 | return cached_get_image_processor( |
| 653 | model_config.model, |
| 654 | revision=model_config.revision, |
| 655 | trust_remote_code=model_config.trust_remote_code, |
| 656 | processor_cls_overrides=processor_cls, |
| 657 | **merged_kwargs, |
| 658 | ) |
| 659 | |
| 660 | def get_hf_processor(self, **kwargs: object): |
| 661 | # AutoProcessor only for tokenizer; its image_processor is resolved by |
| 662 | # class name and can pick the wrong MiniCPM checkpoint. |
| 663 | hf_processor = self.ctx.get_hf_processor(**kwargs) |
| 664 | image_processor = self._get_checkpoint_image_processor(**kwargs) |
| 665 | |
| 666 | from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor |
| 667 | |
| 668 | vendored_processor = MiniCPMVProcessor( |
| 669 | image_processor=image_processor, |
| 670 | tokenizer=hf_processor.tokenizer, |
| 671 | version=self.get_model_version(), |
| 672 | ) |
| 673 | hf_processor = vendored_processor |
| 674 | |
| 675 | # NumPy arrays are considered as Iterable but not Sequence in |
| 676 | # https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428 |
| 677 | # transformers v5+ renamed `mean`/`std` -> `image_mean`/`image_std` |
| 678 | for attr in ("mean", "std", "image_mean", "image_std"): |
| 679 | val = getattr(image_processor, attr, None) |
| 680 | if isinstance(val, np.ndarray): |
| 681 | setattr(image_processor, attr, val.tolist()) |
| 682 | |
| 683 | return hf_processor |
| 684 | |
| 685 | def get_image_processor(self, **kwargs: object): |
| 686 | return self.get_hf_processor(**kwargs).image_processor |
| 687 | |
| 688 | def get_data_parser(self): |
| 689 | return MiniCPMVMultiModalDataParser( |
| 690 | expected_hidden_size=self._get_expected_hidden_size(), |
| 691 | allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, |
| 692 | ) |
| 693 | |
| 694 | def get_model_version(self): |
| 695 | return get_version_by_config(self.get_hf_config()) |
| 696 | |
| 697 | def get_supported_mm_limits(self) -> Mapping[str, int | None]: |
| 698 | mm_limits = {"image": None} |
| 699 | if self.get_model_version() in {(2, 6), (4, 0), (4, 5)}: |
| 700 | mm_limits["video"] = None |
| 701 | |
| 702 | return mm_limits |
| 703 | |
| 704 | def get_slice_image_placeholder( |
| 705 | self, |
| 706 | image_size: ImageSize, |
| 707 | # For MiniCPM V/O 2.6 |
| 708 | image_idx: int = 0, |
| 709 | max_slice_nums: int | None = None, |
| 710 | use_image_id: bool = True, |
| 711 | ) -> str: |
| 712 | image_processor = self.get_image_processor() |
| 713 | version = self.get_model_version() |
| 714 | |
| 715 | if version == (2, 0) or version == (2, 5): |
| 716 | return image_processor.get_slice_image_placeholder(image_size) |
| 717 | |
| 718 | return image_processor.get_slice_image_placeholder( |
| 719 | image_size, |
| 720 | image_idx=image_idx, |
| 721 | max_slice_nums=max_slice_nums, |
| 722 | use_image_id=use_image_id, |
| 723 | ) |
| 724 | |
| 725 | def get_sliced_grid( |
| 726 | self, |
| 727 | image_size: ImageSize, |
| 728 | # For MiniCPM V/O 2.6 |
| 729 | max_slice_nums: int | None = None, |
| 730 | ) -> tuple[int, int] | None: |
| 731 | image_processor = self.get_image_processor() |
| 732 | version = self.get_model_version() |
| 733 | |
| 734 | if version == (2, 0) or version == (2, 5): |
| 735 | return image_processor.get_sliced_grid(image_size) |
| 736 | |
| 737 | if max_slice_nums is None: |
| 738 | max_slice_nums = image_processor.max_slice_nums |
| 739 | |
| 740 | return image_processor.get_sliced_grid( |
| 741 | image_size, |
| 742 | max_slice_nums=max_slice_nums, |
| 743 | ) |
| 744 | |
| 745 | def get_num_image_tokens( |
| 746 | self, |
| 747 | image_size: ImageSize, |
| 748 | max_slice_nums: int | None = None, |
| 749 | ) -> int: |
| 750 | image_processor = self.get_image_processor() |
| 751 | |
| 752 | grid = self.get_sliced_grid( |
| 753 | image_size, |
| 754 | max_slice_nums=max_slice_nums, |
| 755 | ) |
| 756 | |
| 757 | if grid is None: |
| 758 | ncols = nrows = 0 |
| 759 | else: |
| 760 | ncols, nrows = grid |
| 761 | |
| 762 | return (ncols * nrows + 1) * image_processor.image_feature_size |
| 763 | |
| 764 | def get_max_image_tokens(self) -> int: |
| 765 | image_size = self.get_image_size_with_most_features() |
| 766 | return self.get_num_image_tokens(image_size) |
| 767 | |
| 768 | def get_image_max_slice_num(self) -> int: |
| 769 | return getattr(self.get_hf_config(), "max_slice_num", 9) |
| 770 | |
| 771 | def get_image_size_with_most_features(self) -> ImageSize: |
| 772 | image_size = getattr(self.get_hf_config(), "image_size", 448) |
| 773 | max_slice_num = self.get_image_max_slice_num() |
| 774 | return ImageSize(width=image_size, height=image_size * max_slice_num) |
| 775 | |
| 776 | def get_max_video_frame_tokens(self) -> int: |
| 777 | frame_size = self.get_video_frame_size_with_most_features() |
| 778 | |
| 779 | return self.get_num_image_tokens( |
| 780 | frame_size, |
| 781 | max_slice_nums=self.get_video_max_slice_num(), |
| 782 | ) |
| 783 | |
| 784 | def get_max_video_tokens( |
| 785 | self, |
| 786 | seq_len: int, |
| 787 | mm_counts: Mapping[str, int], |
| 788 | ) -> int: |
| 789 | num_frames = self.get_num_frames_with_most_features(seq_len, mm_counts) |
| 790 | num_video_tokens_total = self.get_max_video_frame_tokens() * num_frames |
| 791 | return num_video_tokens_total |
| 792 | |
| 793 | def get_video_max_slice_num(self) -> int: |
| 794 | return 1 |
| 795 | |
| 796 | def get_video_frame_size_with_most_features(self) -> ImageSize: |
| 797 | image_size = getattr(self.get_hf_config(), "image_size", 448) |
| 798 | max_slice_num = self.get_video_max_slice_num() |
| 799 | return ImageSize(width=image_size, height=image_size * max_slice_num) |
| 800 | |
| 801 | def get_max_video_frames(self, max_tokens: int) -> int: |
| 802 | num_frame_tokens = self.get_max_video_frame_tokens() |
| 803 | num_frames = max_tokens // num_frame_tokens |
| 804 | return num_frames |
| 805 | |
| 806 | def get_num_frames_with_most_features( |
| 807 | self, |
| 808 | seq_len: int, |
| 809 | mm_counts: Mapping[str, int], |
| 810 | ) -> int: |
| 811 | max_images = mm_counts.get("image", 0) |
| 812 | max_videos = mm_counts.get("video", 0) |
| 813 | |
| 814 | max_image_tokens = self.get_max_image_tokens() * max_images |
| 815 | max_total_frames = self.get_max_video_frames(seq_len - max_image_tokens) |
| 816 | max_frames_per_video = min( |
| 817 | max_total_frames // max(max_videos, 1), _MAX_FRAMES_PER_VIDEO |
| 818 | ) |
| 819 | |
| 820 | return max(max_frames_per_video, 1) |
| 821 | |
| 822 | def get_mm_max_tokens_per_item( |
| 823 | self, |
| 824 | seq_len: int, |
| 825 | mm_counts: Mapping[str, int], |
| 826 | ) -> Mapping[str, int]: |
| 827 | max_tokens = {"image": self.get_max_image_tokens()} |
| 828 | if "video" in self.get_supported_mm_limits(): |
| 829 | max_tokens["video"] = self.get_max_video_tokens(seq_len, mm_counts) |
| 830 | return max_tokens |
| 831 | |
| 832 | |
| 833 | _I = TypeVar("_I", bound=MiniCPMVProcessingInfo, default=MiniCPMVProcessingInfo) |
| 834 | |
| 835 | |
| 836 | class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]): |
| 837 | def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: |
| 838 | num_images = mm_counts.get("image", 0) |
| 839 | num_videos = mm_counts.get("video", 0) |
| 840 | |
| 841 | image_prompt_texts = self.info.image_pattern * num_images |
| 842 | video_prompt_texts = self.info.video_pattern * num_videos |
| 843 | |
| 844 | return image_prompt_texts + video_prompt_texts |
| 845 | |
| 846 | def get_dummy_mm_data( |
| 847 | self, |
| 848 | seq_len: int, |
| 849 | mm_counts: Mapping[str, int], |
| 850 | mm_options: MultiModalDummyOptions, |
| 851 | ) -> MultiModalDataDict: |
| 852 | num_videos = mm_counts.get("video", 0) |
| 853 | |
| 854 | image_width, image_height = self.info.get_image_size_with_most_features() |
| 855 | video_width, video_height = self.info.get_video_frame_size_with_most_features() |
| 856 | num_video_frames = self.info.get_num_frames_with_most_features( |
| 857 | seq_len, mm_counts |
| 858 | ) |
| 859 | |
| 860 | video_overrides = mm_options.get("video") |
| 861 | |
| 862 | # Convert video overrides to image overrides for per-frame image generation, |
| 863 | # and apply num_frames override to num_video_frames. |
| 864 | video_frame_overrides: ImageDummyOptions | None = None |
| 865 | if video_overrides is not None: |
| 866 | if video_overrides.num_frames: |
| 867 | num_video_frames = min(num_video_frames, video_overrides.num_frames) |
| 868 | if video_overrides.width or video_overrides.height: |
| 869 | video_frame_overrides = ImageDummyOptions( |
| 870 | count=video_overrides.count, |
| 871 | width=video_overrides.width, |
| 872 | height=video_overrides.height, |
| 873 | ) |
| 874 | |
| 875 | return { |
| 876 | "image": self._get_dummy_images( |
| 877 | width=image_width, |
| 878 | height=image_height, |
| 879 | num_images=mm_counts.get("image", 0), |
| 880 | overrides=mm_options.get("image"), |
| 881 | ), |
| 882 | "video": [ |
| 883 | self._get_dummy_images( |
| 884 | width=video_width, |
| 885 | height=video_height, |
| 886 | num_images=num_video_frames, |
| 887 | overrides=video_frame_overrides, |
| 888 | ) |
| 889 | ] |
| 890 | * num_videos, |
| 891 | } |
| 892 | |
| 893 | |
| 894 | class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]): |
| 895 | def get_image_prompt_texts(self, image_size: ImageSize, image_idx: int = 0) -> str: |
| 896 | return self.info.get_slice_image_placeholder( |
| 897 | image_size, |
| 898 | image_idx=image_idx, |
| 899 | ) |
| 900 | |
| 901 | def get_video_prompt_texts(self, image_size: ImageSize, num_frames: int) -> str: |
| 902 | return ( |
| 903 | self.info.get_slice_image_placeholder( |
| 904 | image_size=image_size, |
| 905 | image_idx=0, |
| 906 | max_slice_nums=self.info.get_video_max_slice_num(), |
| 907 | use_image_id=False, |
| 908 | ) |
| 909 | * num_frames |
| 910 | ) |
| 911 | |
| 912 | def process_images( |
| 913 | self, |
| 914 | mm_data: Mapping[str, object], |
| 915 | mm_kwargs: Mapping[str, object], |
| 916 | ) -> Mapping[str, NestedTensors]: |
| 917 | if (images := mm_data.get("images")) is None: |
| 918 | return {} |
| 919 | |
| 920 | mm_items = self.info.parse_mm_data({"image": images}, validate=False) |
| 921 | parsed_images = mm_items.get_items( |
| 922 | "image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems) |
| 923 | ) |
| 924 | |
| 925 | if isinstance(parsed_images, MiniCPMVImageEmbeddingItems): |
| 926 | image_inputs = {} |
| 927 | else: |
| 928 | image_inputs = self._call_hf_processor_on_prompts( |
| 929 | prompts=[self.info.image_pattern] * len(parsed_images), |
| 930 | mm_data={"images": [[image] for image in parsed_images]}, |
| 931 | mm_kwargs=mm_kwargs, |
| 932 | out_keys={"pixel_values", "image_sizes", "tgt_sizes"}, |
| 933 | ) |
| 934 | |
| 935 | return image_inputs |
| 936 | |
| 937 | def process_videos( |
| 938 | self, |
| 939 | mm_data: Mapping[str, object], |
| 940 | mm_kwargs: Mapping[str, object], |
| 941 | ) -> Mapping[str, NestedTensors]: |
| 942 | if (videos := mm_data.get("videos")) is None: |
| 943 | return {} |
| 944 | |
| 945 | mm_items = self.info.parse_mm_data({"video": videos}, validate=False) |
| 946 | parsed_videos = mm_items.get_items( |
| 947 | "video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems) |
| 948 | ) |
| 949 | |
| 950 | if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems): |
| 951 | video_inputs = {} |
| 952 | else: |
| 953 | video_inputs = self._call_hf_processor_on_prompts( |
| 954 | prompts=[ |
| 955 | self.info.image_pattern * len(video) for video in parsed_videos |
| 956 | ], |
| 957 | mm_data={"images": list(parsed_videos)}, |
| 958 | mm_kwargs={ |
| 959 | **mm_kwargs, |
| 960 | "max_slice_nums": self.info.get_video_max_slice_num(), |
| 961 | }, |
| 962 | out_keys={"pixel_values", "image_sizes", "tgt_sizes"}, |
| 963 | ) |
| 964 | |
| 965 | video_inputs = {f"video_{k}": v for k, v in video_inputs.items()} |
| 966 | |
| 967 | return video_inputs |
| 968 | |
| 969 | def process_mm_inputs( |
| 970 | self, |
| 971 | mm_data: Mapping[str, object], |
| 972 | mm_kwargs: Mapping[str, object], |
| 973 | ) -> Mapping[str, NestedTensors]: |
| 974 | return { |
| 975 | **self.process_images(mm_data, mm_kwargs), |
| 976 | **self.process_videos(mm_data, mm_kwargs), |
| 977 | } |
| 978 | |
| 979 | def _apply_prompt_updates( |
| 980 | self, |
| 981 | token_ids: list[int], |
| 982 | mm_prompt_updates: MultiModalPromptUpdates, |
| 983 | ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: |
| 984 | """Apply multi-modal prompt updates to token IDs.""" |
| 985 | new_token_ids, match_result = self._apply_token_matches( |
| 986 | token_ids, |
| 987 | mm_prompt_updates, |
| 988 | ) |
| 989 | |
| 990 | # If the target does not consist of special tokens, it may be |
| 991 | # tokenized differently inside the prompt, so fall back to |
| 992 | # performing the updates on the decoded text, then encoding the |
| 993 | # result back. The segments are encoded separately to avoid BPE |
| 994 | # merges across segment boundaries. |
| 995 | if not all( |
| 996 | all(update_idx is not None for update_idx in update_idxs) |
| 997 | for update_idxs in match_result.values() |
| 998 | ): |
| 999 | new_token_ids, match_result = self._apply_prompt_updates_via_text( |
| 1000 | token_ids, |
| 1001 | mm_prompt_updates, |
| 1002 | encode_segments_separately=True, |
| 1003 | ) |
| 1004 | |
| 1005 | matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list) |
| 1006 | for modality, update_idxs in match_result.items(): |
| 1007 | for item_idx, update_idx in enumerate(update_idxs): |
| 1008 | assert update_idx is not None, ( |
| 1009 | "Failed to apply prompt replacement for " |
| 1010 | f"mm_items[{modality!r}][{item_idx}]" |
| 1011 | ) |
| 1012 | |
| 1013 | matched_updates[modality].append( |
| 1014 | [mm_prompt_updates[modality][item_idx][update_idx]] |
| 1015 | ) |
| 1016 | |
| 1017 | placeholders = self._find_mm_placeholders( |
| 1018 | new_token_ids, |
| 1019 | dict(matched_updates), |
| 1020 | ) |
| 1021 | |
| 1022 | return new_token_ids, placeholders |
| 1023 | |
| 1024 | def _call_hf_processor_on_prompts( |
| 1025 | self, |
| 1026 | prompts: list[str], |
| 1027 | mm_data: Mapping[str, Sequence[object]], |
| 1028 | mm_kwargs: Mapping[str, object], |
| 1029 | *, |
| 1030 | out_keys: set[str], |
| 1031 | ) -> dict[str, NestedTensors]: |
| 1032 | # This processor supports zipping prompt and mm_data together |
| 1033 | if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5), (4, 6)}: |
| 1034 | inputs = self.info.ctx.call_hf_processor( |
| 1035 | self.info.get_hf_processor(**mm_kwargs), |
| 1036 | dict(text=prompts, **mm_data), |
| 1037 | mm_kwargs, |
| 1038 | ) |
| 1039 | else: |
| 1040 | inputs = defaultdict[str, list[torch.Tensor]](list) |
| 1041 | |
| 1042 | for i, prompt in enumerate(prompts): |
| 1043 | inputs_one = self.info.ctx.call_hf_processor( |
| 1044 | self.info.get_hf_processor(**mm_kwargs), |
| 1045 | dict(text=prompt, **{k: v[i] for k, v in mm_data.items()}), |
| 1046 | mm_kwargs, |
| 1047 | ) |
| 1048 | |
| 1049 | for k, v in inputs_one.items(): |
| 1050 | assert len(v) == 1, (k, len(v)) |
| 1051 | inputs[k].append(v[0]) |
| 1052 | |
| 1053 | return {k: inputs[k] for k in out_keys} |
| 1054 | |
| 1055 | def _apply_hf_processor_main( |
| 1056 | self, |
| 1057 | mm_items: MultiModalDataItems, |
| 1058 | hf_kwargs: Mapping[str, object], |
| 1059 | ) -> BatchFeature: |
| 1060 | mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( |
| 1061 | mm_items, hf_kwargs |
| 1062 | ) |
| 1063 | |
| 1064 | prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) |
| 1065 | |
| 1066 | tokenizer = self.info.get_tokenizer() |
| 1067 | |
| 1068 | input_ids = torch.tensor([tokenizer.encode(prompt_text)]) |
| 1069 | mm_inputs = self.process_mm_inputs(mm_data, hf_kwargs) |
| 1070 | |
| 1071 | processed_data = BatchFeature( |
| 1072 | { |
| 1073 | "input_ids": input_ids, |
| 1074 | **mm_inputs, |
| 1075 | } |
| 1076 | ) |
| 1077 | return self._finalize_hf_mm_data( |
| 1078 | mm_data, hf_kwargs, passthrough_data, processed_data |
| 1079 | ) |
| 1080 | |
| 1081 | def _get_prompt_updates( |
| 1082 | self, |
| 1083 | mm_items: MultiModalDataItems, |
| 1084 | hf_processor_mm_kwargs: Mapping[str, object], |
| 1085 | out_mm_kwargs: MultiModalKwargsItems, |
| 1086 | ) -> Sequence[PromptUpdate]: |
| 1087 | placeholders = [ |
| 1088 | ("image", self.info.image_pattern), |
| 1089 | ("video", self.info.video_pattern), |
| 1090 | ] |
| 1091 | |
| 1092 | # hard code for inconsistency of encode-decode image_pattern |
| 1093 | additional_placeholders = [] |
| 1094 | tokenizer = self.info.get_tokenizer() |
| 1095 | for modality, pattern in placeholders: |
| 1096 | sub_pattern = tokenizer.decode( |
| 1097 | cached_encode(tokenizer, pattern, add_special_tokens=False) |
| 1098 | ) |
| 1099 | if sub_pattern != pattern: |
| 1100 | additional_placeholders.append((modality, sub_pattern)) |
| 1101 | placeholders += additional_placeholders |
| 1102 | |
| 1103 | vocab = tokenizer.get_vocab() |
| 1104 | unk_token_ids = [vocab["<unk>"]] |
| 1105 | |
| 1106 | def get_image_replacement(item_idx: int): |
| 1107 | images = mm_items.get_items( |
| 1108 | "image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems) |
| 1109 | ) |
| 1110 | assert isinstance( |
| 1111 | images, (MiniCPMVImageEmbeddingItems, ImageProcessorItems) |
| 1112 | ) |
| 1113 | |
| 1114 | image_size = images.get_image_size(item_idx) |
| 1115 | |
| 1116 | return PromptUpdateDetails.select_token_ids( |
| 1117 | cached_encode( |
| 1118 | tokenizer, |
| 1119 | self.get_image_prompt_texts(image_size, item_idx), |
| 1120 | add_special_tokens=False, |
| 1121 | ), |
| 1122 | unk_token_ids, |
| 1123 | ) |
| 1124 | |
| 1125 | def get_video_replacement(item_idx: int): |
| 1126 | videos = mm_items.get_items( |
| 1127 | "video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems) |
| 1128 | ) |
| 1129 | assert isinstance( |
| 1130 | videos, (MiniCPMVVideoEmbeddingItems, VideoProcessorItems) |
| 1131 | ) |
| 1132 | |
| 1133 | frame_size = videos.get_frame_size(item_idx) |
| 1134 | num_frames = videos.get_num_frames(item_idx) |
| 1135 | |
| 1136 | return PromptUpdateDetails.select_token_ids( |
| 1137 | cached_encode( |
| 1138 | tokenizer, |
| 1139 | self.get_video_prompt_texts(frame_size, num_frames), |
| 1140 | add_special_tokens=False, |
| 1141 | ), |
| 1142 | unk_token_ids, |
| 1143 | ) |
| 1144 | |
| 1145 | get_replacement = { |
| 1146 | "image": get_image_replacement, |
| 1147 | "video": get_video_replacement, |
| 1148 | } |
| 1149 | |
| 1150 | return [ |
| 1151 | PromptReplacement( |
| 1152 | modality=modality, |
| 1153 | target=cached_encode(tokenizer, pattern, add_special_tokens=False), |
| 1154 | replacement=get_replacement[modality], |
| 1155 | ) |
| 1156 | for modality, pattern in placeholders |
| 1157 | ] |
| 1158 | |
| 1159 | def _recompute_cached_prompt_update( |
| 1160 | self, |
| 1161 | cached_update: ResolvedPromptUpdate, |
| 1162 | new_item_idx: int, |
| 1163 | ) -> ResolvedPromptUpdate: |
| 1164 | new_update = super()._recompute_cached_prompt_update( |
| 1165 | cached_update, |
| 1166 | new_item_idx, |
| 1167 | ) |
| 1168 | |
| 1169 | if cached_update.modality == "image": |
| 1170 | tokenizer = self.info.get_tokenizer() |
| 1171 | image_processor = self.info.get_image_processor() |
| 1172 | version = self.info.get_model_version() |
| 1173 | |
| 1174 | text = tokenizer.decode(cached_update.content.full) |
| 1175 | prev_item_idx = cached_update.item_idx |
| 1176 | |
| 1177 | if version == (2, 0) or version == (2, 5): |
| 1178 | im_start = image_processor.im_start_token |
| 1179 | im_end = image_processor.im_end_token |
| 1180 | elif hasattr(image_processor, "im_id_start"): |
| 1181 | im_start = image_processor.im_id_start |
| 1182 | im_end = image_processor.im_id_end |
| 1183 | else: |
| 1184 | # transformers v5.7+ keeps im_id tokens on the tokenizer. |
| 1185 | im_start = getattr(tokenizer, "image_id_start_token", "<image_id>") |
| 1186 | im_end = getattr(tokenizer, "image_id_end_token", "</image_id>") |
| 1187 | |
| 1188 | embed_text = getattr(tokenizer, "image_token", "<unk>") |
| 1189 | new_update = new_update.with_content( |
| 1190 | PromptUpdateDetails.select_token_ids( |
| 1191 | cached_encode( |
| 1192 | tokenizer, |
| 1193 | text.replace( |
| 1194 | f"{im_start}{prev_item_idx}{im_end}", |
| 1195 | f"{im_start}{new_item_idx}{im_end}", |
| 1196 | 1, |
| 1197 | ), |
| 1198 | add_special_tokens=False, |
| 1199 | ), |
| 1200 | cached_encode(tokenizer, embed_text, add_special_tokens=False), |
| 1201 | ) |
| 1202 | ) |
| 1203 | |
| 1204 | return new_update |
| 1205 | |
| 1206 | def _get_mm_fields_config( |
| 1207 | self, |
| 1208 | hf_inputs: BatchFeature, |
| 1209 | hf_processor_mm_kwargs: Mapping[str, object], |
| 1210 | ) -> Mapping[str, MultiModalFieldConfig]: |
| 1211 | return _minicpmv_field_config(hf_inputs) |
| 1212 | |
| 1213 | |
| 1214 | class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP): |
| 1215 | """The abstract class of MiniCPMV can only be inherited, but cannot be |
| 1216 | instantiated. |
| 1217 | """ |
| 1218 | |
| 1219 | supports_encoder_tp_data = True |
| 1220 | |
| 1221 | @classmethod |
| 1222 | def get_placeholder_str(cls, modality: str, i: int) -> str | None: |
| 1223 | if modality.startswith("image"): |
| 1224 | return "(<image>./</image>)" |
| 1225 | if modality.startswith("video"): |
| 1226 | return "(<video>./</video>)" |
| 1227 | |
| 1228 | raise ValueError("Only image or video modality is supported") |
| 1229 | |
| 1230 | def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): |
| 1231 | config = vllm_config.model_config.hf_config |
| 1232 | multimodal_config = vllm_config.model_config.multimodal_config |
| 1233 | assert multimodal_config is not None |
| 1234 | quant_config = vllm_config.quant_config |
| 1235 | self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" |
| 1236 | super().__init__() |
| 1237 | # All MiniCPM-V models disable `tie_word_embeddings` but |
| 1238 | # `PreTrainedConfig.tie_word_embeddings` defaults to True; we cannot |
| 1239 | # check `tie_word_embeddings` until vLLM integrate MiniCPM-V model |
| 1240 | # and config class |
| 1241 | self.config = config |
| 1242 | self.multimodal_config = multimodal_config |
| 1243 | self.vllm_config = vllm_config |
| 1244 | |
| 1245 | self.version = get_version_by_config(self.config) |
| 1246 | |
| 1247 | with self._mark_language_model(vllm_config): |
| 1248 | self.llm = self.init_llm( |
| 1249 | vllm_config=vllm_config, prefix=maybe_prefix(prefix, "llm") |
| 1250 | ) |
| 1251 | |
| 1252 | with self._mark_tower_model(vllm_config, {"image", "video"}): |
| 1253 | self.vpm = self.init_vision_module( |
| 1254 | config, quant_config, prefix=maybe_prefix(prefix, "vpm") |
| 1255 | ) |
| 1256 | self.vision_dim = ( |
| 1257 | self.vpm.embed_dim |
| 1258 | if self.version == (2, 0) |
| 1259 | else self.vpm.embeddings.embed_dim |
| 1260 | ) |
| 1261 | self.embed_dim = self.config.hidden_size |
| 1262 | |
| 1263 | self.resampler = self.init_resampler( |
| 1264 | self.embed_dim, |
| 1265 | self.vision_dim, |
| 1266 | quant_config=quant_config, |
| 1267 | prefix=maybe_prefix(prefix, "resampler"), |
| 1268 | ) |
| 1269 | self._resampler_moved = False |
| 1270 | |
| 1271 | self.make_empty_intermediate_tensors = self.llm.make_empty_intermediate_tensors |
| 1272 | |
| 1273 | def _ensure_resampler_device(self) -> None: |
| 1274 | if self._resampler_moved: |
| 1275 | return |
| 1276 | # Only move device, DO NOT touch dtype (fp8 quant needs its own dtype) |
| 1277 | self.resampler.to(current_platform.device_type) |
| 1278 | self._resampler_moved = True |
| 1279 | |
| 1280 | def _parse_and_validate_vision_input( |
| 1281 | self, |
| 1282 | modality: str, |
| 1283 | **kwargs: object, |
| 1284 | ) -> MiniCPMVImageInputs | None: |
| 1285 | pixel_values = kwargs.pop("pixel_values", None) |
| 1286 | image_embeds = kwargs.pop("image_embeds", None) |
| 1287 | |
| 1288 | if pixel_values is None and image_embeds is None: |
| 1289 | return None |
| 1290 | |
| 1291 | if image_embeds is not None: |
| 1292 | return MiniCPMVImageEmbeddingInputs( |
| 1293 | type="image_embeds", |
| 1294 | image_embeds=image_embeds, |
| 1295 | ) |
| 1296 | |
| 1297 | assert is_list_of(pixel_values, list, check="all") |
| 1298 | pixel_values_list: list[list[torch.Tensor]] = [] |
| 1299 | for pixel_value in pixel_values: |
| 1300 | assert is_list_of(pixel_value, torch.Tensor, check="all") |
| 1301 | pixel_values_list.append(pixel_value) |
| 1302 | tgt_sizes = kwargs.pop("tgt_sizes") |
| 1303 | |
| 1304 | num_slices_flat = torch.tensor([len(ps) for ps in pixel_values_list]) |
| 1305 | pixel_values_flat = flatten_2d_lists(pixel_values_list) |
| 1306 | tgt_sizes_flat = flatten_bn(tgt_sizes, concat=True) |
| 1307 | |
| 1308 | return MiniCPMVImagePixelInputs( |
| 1309 | type="pixel_values", |
| 1310 | pixel_values=pixel_values_flat, |
| 1311 | tgt_sizes=tgt_sizes_flat, |
| 1312 | num_slices=num_slices_flat, |
| 1313 | ) |
| 1314 | |
| 1315 | def _parse_and_validate_multimodal_inputs( |
| 1316 | self, **kwargs: object |
| 1317 | ) -> MiniCPMVMultiModalInputs: |
| 1318 | kwargs.pop("modality", None) |
| 1319 | modalities: MiniCPMVMultiModalInputs = {} |
| 1320 | |
| 1321 | # Preserve the order of modalities if there are multiple of them |
| 1322 | # from the order of kwargs. |
| 1323 | for input_key in kwargs: |
| 1324 | if ( |
| 1325 | input_key in ("pixel_values", "image_embeds") |
| 1326 | and "images" not in modalities |
| 1327 | ): |
| 1328 | modalities["images"] = self._parse_and_validate_vision_input( |
| 1329 | "images", **kwargs |
| 1330 | ) |
| 1331 | if ( |
| 1332 | input_key in ("video_pixel_values", "video_embeds") |
| 1333 | and "videos" not in modalities |
| 1334 | ): |
| 1335 | modalities["videos"] = self._parse_and_validate_vision_input( |
| 1336 | "videos", **_image_kwargs_from_video(kwargs) |
| 1337 | ) |
| 1338 | |
| 1339 | return modalities |
| 1340 | |
| 1341 | def _process_vision_input( |
| 1342 | self, |
| 1343 | image_input: MiniCPMVImageInputs, |
| 1344 | ) -> torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...]: |
| 1345 | if image_input["type"] == "image_embeds": |
| 1346 | return image_input["image_embeds"] |
| 1347 | |
| 1348 | assert isinstance(image_input, MiniCPMVImagePixelInputs) |
| 1349 | image_features_flat = self.get_vision_hidden_states(image_input) |
| 1350 | |
| 1351 | num_slices = image_input["num_slices"] |
| 1352 | return [e.flatten(0, 1) for e in image_features_flat.split(num_slices.tolist())] |
| 1353 | |
| 1354 | def _process_multimodal_inputs(self, modalities: MiniCPMVMultiModalInputs): |
| 1355 | # The result multimodal_embeddings is tuple of tensors, with each |
| 1356 | # tensor corresponding to a multimodal data item (image or video). |
| 1357 | multimodal_embeddings: tuple[torch.Tensor, ...] = () |
| 1358 | |
| 1359 | # NOTE: It is important to iterate over the keys in this dictionary |
| 1360 | # to preserve the order of the modalities. |
| 1361 | for modality in modalities: |
| 1362 | if modality == "images": |
| 1363 | image_input = modalities["images"] |
| 1364 | assert image_input is not None |
| 1365 | image_embeddings = self._process_vision_input(image_input) |
| 1366 | multimodal_embeddings += tuple(image_embeddings) |
| 1367 | if modality == "videos": |
| 1368 | video_input = modalities["videos"] |
| 1369 | assert video_input is not None |
| 1370 | video_embeddings = self._process_vision_input(video_input) |
| 1371 | multimodal_embeddings += tuple(video_embeddings) |
| 1372 | |
| 1373 | return multimodal_embeddings |
| 1374 | |
| 1375 | def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: |
| 1376 | modalities = self._parse_and_validate_multimodal_inputs(**kwargs) |
| 1377 | if not modalities: |
| 1378 | return [] |
| 1379 | |
| 1380 | return self._process_multimodal_inputs(modalities) |
| 1381 | |
| 1382 | def forward( |
| 1383 | self, |
| 1384 | input_ids: torch.Tensor | None, |
| 1385 | positions: torch.Tensor, |
| 1386 | intermediate_tensors: IntermediateTensors | None = None, |
| 1387 | inputs_embeds: torch.Tensor | None = None, |
| 1388 | **kwargs: Any, |
| 1389 | ) -> torch.Tensor: |
| 1390 | if intermediate_tensors is not None: |
| 1391 | inputs_embeds = None |
| 1392 | |
| 1393 | hidden_states = self.llm.model( |
| 1394 | input_ids=input_ids, |
| 1395 | positions=positions, |
| 1396 | intermediate_tensors=intermediate_tensors, |
| 1397 | inputs_embeds=inputs_embeds, |
| 1398 | ) |
| 1399 | return hidden_states |
| 1400 | |
| 1401 | def compute_logits( |
| 1402 | self, |
| 1403 | hidden_states: torch.Tensor, |
| 1404 | ) -> torch.Tensor | None: |
| 1405 | return self.llm.compute_logits(hidden_states) |
| 1406 | |
| 1407 | def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: |
| 1408 | loader = AutoWeightsLoader(self) |
| 1409 | loaded = loader.load_weights(weights) |
| 1410 | self._ensure_resampler_device() |
| 1411 | return loaded |
| 1412 | |
| 1413 | def get_mm_mapping(self) -> MultiModelKeys: |
| 1414 | """Get the module prefix in multimodal models.""" |
| 1415 | return MultiModelKeys.from_string_field( |
| 1416 | language_model="llm", connector="resampler", tower_model="vpm" |
| 1417 | ) |
| 1418 | |
| 1419 | def init_llm( |
| 1420 | self, |
| 1421 | vllm_config: VllmConfig, |
| 1422 | prefix: str = "", |
| 1423 | ) -> nn.Module: |
| 1424 | raise NotImplementedError |
| 1425 | |
| 1426 | def init_vision_module( |
| 1427 | self, |
| 1428 | config: PreTrainedConfig, |
| 1429 | quant_config: QuantizationConfig | None, |
| 1430 | prefix: str = "", |
| 1431 | ) -> nn.Module: |
| 1432 | raise NotImplementedError |
| 1433 | |
| 1434 | def init_resampler( |
| 1435 | self, |
| 1436 | embed_dim: int, |
| 1437 | vision_dim: int, |
| 1438 | quant_config: QuantizationConfig | None = None, |
| 1439 | prefix: str = "", |
| 1440 | ) -> nn.Module: |
| 1441 | raise NotImplementedError |
| 1442 | |
| 1443 | def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor: |
| 1444 | raise NotImplementedError |
| 1445 | |
| 1446 | |
| 1447 | # --- Encoder CUDA graph (MiniCPM-V 2.5 / 2.6 / 4.0; mixed into subclasses) --- |
| 1448 | # Buffer keys |
| 1449 | _MINICPMV_CUDAGRAPH_BUF_KEY_TGT_SIZES = "minicpmv_tgt_sizes" |
| 1450 | _MINICPMV_CUDAGRAPH_BUF_KEY_PATCH_MASK = "minicpmv_patch_attn_mask" |
| 1451 | _MINICPMV_CUDAGRAPH_BUF_KEY_PIXEL = "minicpmv_pixel_values" |
| 1452 | |
| 1453 | # mm_kwargs keys for the flat pixel-value tensor |
| 1454 | _MINICPMV_CUDAGRAPH_FLAT_KEY_IMAGE = "minicpmv_encoder_input_flat" |
| 1455 | _MINICPMV_CUDAGRAPH_FLAT_KEY_VIDEO = "minicpmv_video_encoder_input_flat" |
| 1456 | # Resolved patch-grid capture-axis key ``(nb_h, nb_w)`` of the sub-batch; |
| 1457 | # pixel dims and max patches are derived from it and the patch size. |
| 1458 | _MINICPMV_CUDAGRAPH_PATCH_GRID_KEY = "minicpmv_encoder_patch_grid" |
| 1459 | |
| 1460 | _MINICPMV_BASE_PATCH_BUCKETS: tuple[tuple[int, int], ...] = ( |
| 1461 | (32, 32), |
| 1462 | (32, 64), |
| 1463 | (64, 32), |
| 1464 | ) |
| 1465 | |
| 1466 | _ENCODER_CUDAGRAPH_MM_KWARGS_SKIP_KEYS = frozenset( |
| 1467 | { |
| 1468 | _MINICPMV_CUDAGRAPH_FLAT_KEY_IMAGE, |
| 1469 | _MINICPMV_CUDAGRAPH_FLAT_KEY_VIDEO, |
| 1470 | _MINICPMV_CUDAGRAPH_PATCH_GRID_KEY, |
| 1471 | } |
| 1472 | ) |
| 1473 | |
| 1474 | |
| 1475 | def _mcpmv_tgt_sizes_tensor( |
| 1476 | mm_kwargs: dict[str, Any], *, video: bool |
| 1477 | ) -> torch.Tensor | list[torch.Tensor]: |
| 1478 | key = "video_tgt_sizes" if video else "tgt_sizes" |
| 1479 | return mm_kwargs[key] |
| 1480 | |
| 1481 | |
| 1482 | def _mcpmv_normalize_tgt_sizes( |
| 1483 | tgt_sizes: torch.Tensor | list[torch.Tensor], |
| 1484 | slice_counts: list[int], |
| 1485 | ) -> torch.Tensor: |
| 1486 | """Normalize tgt_sizes to shape ``(total_slices, 2)``.""" |
| 1487 | if isinstance(tgt_sizes, list): |
| 1488 | if not tgt_sizes: |
| 1489 | tgt_sizes = torch.zeros((0, 2), dtype=torch.long) |
| 1490 | else: |
| 1491 | tgt_sizes = torch.cat(tgt_sizes, dim=0) |
| 1492 | |
| 1493 | total_slices = sum(slice_counts) |
| 1494 | if tgt_sizes.dim() == 2 and tgt_sizes.shape[0] == total_slices: |
| 1495 | return tgt_sizes |
| 1496 | if tgt_sizes.dim() == 3: |
| 1497 | return torch.cat( |
| 1498 | [tgt_sizes[i, : slice_counts[i], :] for i in range(len(slice_counts))], |
| 1499 | dim=0, |
| 1500 | ) |
| 1501 | return torch.repeat_interleave( |
| 1502 | tgt_sizes, |
| 1503 | torch.tensor(slice_counts, device=tgt_sizes.device), |
| 1504 | dim=0, |
| 1505 | ) |
| 1506 | |
| 1507 | |
| 1508 | def _mcpmv_pack_flat_pixels( |
| 1509 | slices: list[torch.Tensor], |
| 1510 | *, |
| 1511 | pixel_height: int, |
| 1512 | pixel_width: int, |
| 1513 | max_num_slices: int, |
| 1514 | device: torch.device, |
| 1515 | dtype: torch.dtype, |
| 1516 | patch_size: int, |
| 1517 | tgt_sizes: torch.Tensor, |
| 1518 | ) -> torch.Tensor: |
| 1519 | """Pack slice tensors into a fixed ``(max_num_slices, 3*H*W)`` buffer.""" |
| 1520 | import torch.nn.functional as F |
| 1521 | |
| 1522 | flat_dim = 3 * pixel_height * pixel_width |
| 1523 | grid_h = pixel_height // patch_size |
| 1524 | grid_w = pixel_width // patch_size |
| 1525 | max_grid_patches = grid_h * grid_w |
| 1526 | |
| 1527 | packed = torch.zeros((max_num_slices, flat_dim), device=device, dtype=dtype) |
| 1528 | n = min(len(slices), max_num_slices) |
| 1529 | tgt_list = tgt_sizes.tolist() |
| 1530 | for i, slc in enumerate(slices[:n]): |
| 1531 | slc = slc.to(dtype=dtype, device=device) |
| 1532 | C, h, w = slc.shape |
| 1533 | if h == patch_size: |
| 1534 | nb_h, nb_w = int(tgt_list[i][0]), int(tgt_list[i][1]) |
| 1535 | num_patches = nb_h * nb_w |
| 1536 | |
| 1537 | patches = ( |
| 1538 | slc.view(C, patch_size, nb_h, nb_w, patch_size) |
| 1539 | .permute(0, 2, 3, 1, 4) |
| 1540 | .reshape(C, num_patches, patch_size, patch_size) |
| 1541 | ) |
| 1542 | |
| 1543 | col = patches.permute(0, 2, 3, 1).reshape( |
| 1544 | 1, C * patch_size * patch_size, num_patches |
| 1545 | ) |
| 1546 | col_padded = col.new_zeros(1, C * patch_size * patch_size, max_grid_patches) |
| 1547 | col_padded[..., :num_patches] = col |
| 1548 | |
| 1549 | folded = F.fold( |
| 1550 | col_padded, |
| 1551 | output_size=(pixel_height, pixel_width), |
| 1552 | kernel_size=patch_size, |
| 1553 | stride=patch_size, |
| 1554 | ) |
| 1555 | packed[i] = folded.reshape(-1) |
| 1556 | else: |
| 1557 | packed[i].view(3, pixel_height, pixel_width)[:, :h, :w] = slc |
| 1558 | return packed |
| 1559 | |
| 1560 | |
| 1561 | class _MiniCPMVEncoderCudaGraphMixin(MiniCPMVBaseModel, SupportsEncoderCudaGraph): |
| 1562 | """SupportsEncoderCudaGraph for MiniCPM-V Idefics2 + resampler (not 2.0).""" |
| 1563 | |
| 1564 | supports_encoder_cudagraph: ClassVar[Literal[True]] = True |
| 1565 | |
| 1566 | def _mcpmv_slice_pixel_size(self) -> tuple[int, int]: |
| 1567 | image_size = int(self.vpm.embeddings.image_size) |
| 1568 | return image_size, image_size |
| 1569 | |
| 1570 | def _mcpmv_patch_grid_pixel_hw( |
| 1571 | self, patch_grid_key: tuple[int, int] |
| 1572 | ) -> tuple[int, int]: |
| 1573 | patch_size = int(self.vpm.embeddings.patch_size) |
| 1574 | th, tw = patch_grid_key |
| 1575 | return th * patch_size, tw * patch_size |
| 1576 | |
| 1577 | def _mcpmv_patch_grid_num_patches(self, patch_grid_key: tuple[int, int]) -> int: |
| 1578 | th, tw = patch_grid_key |
| 1579 | return th * tw |
| 1580 | |
| 1581 | def _mcpmv_patch_grid_keys(self) -> tuple[tuple[int, int], ...]: |
| 1582 | """Ordered patch-grid keys ``(nb_h, nb_w)`` for the capture axis.""" |
| 1583 | max_side = int(self.vpm.embeddings.image_size) // int( |
| 1584 | self.vpm.embeddings.patch_size |
| 1585 | ) |
| 1586 | keys: list[tuple[int, int]] = [] |
| 1587 | seen: set[tuple[int, int]] = set() |
| 1588 | for th, tw in _MINICPMV_BASE_PATCH_BUCKETS: |
| 1589 | if th <= max_side and tw <= max_side: |
| 1590 | keys.append((th, tw)) |
| 1591 | seen.add((th, tw)) |
| 1592 | full = (max_side, max_side) |
| 1593 | if full not in seen: |
| 1594 | keys.append(full) |
| 1595 | return tuple(keys) |
| 1596 | |
| 1597 | def _mcpmv_resolve_patch_grid( |
| 1598 | self, |
| 1599 | tgt_sizes: torch.Tensor, |
| 1600 | slice_counts: list[int], |
| 1601 | indices: list[int], |
| 1602 | ) -> tuple[int, int]: |
| 1603 | """Smallest patch-grid key covering all selected items.""" |
| 1604 | keys = self._mcpmv_patch_grid_keys() |
| 1605 | if not indices: |
| 1606 | return keys[0] |
| 1607 | tgt_groups = torch.split(tgt_sizes, slice_counts) |
| 1608 | selected = torch.cat([tgt_groups[i] for i in indices], dim=0) |
| 1609 | need_h = int(selected[:, 0].max().item()) |
| 1610 | need_w = int(selected[:, 1].max().item()) |
| 1611 | for th, tw in keys: |
| 1612 | if th >= need_h and tw >= need_w: |
| 1613 | return (th, tw) |
| 1614 | return keys[-1] |
| 1615 | |
| 1616 | def _mcpmv_max_slices_cap( |
| 1617 | self, |
| 1618 | token_budget: int, |
| 1619 | max_batch_size: int, |
| 1620 | max_frames_per_batch: int, |
| 1621 | ) -> int: |
| 1622 | max_slice_num = int(getattr(self.config, "max_slice_num", 9)) |
| 1623 | query_num = max(1, int(self.config.query_num)) |
| 1624 | max_slices_by_token_budget = max(1, token_budget // query_num) |
| 1625 | max_slices_by_content = max_batch_size * (max_slice_num + 1) |
| 1626 | if self.version in {(2, 6), (4, 0)} and max_frames_per_batch > 0: |
| 1627 | max_slices_by_content = max( |
| 1628 | max_slices_by_content, |
| 1629 | max_frames_per_batch * (max_slice_num + 1), |
| 1630 | ) |
| 1631 | return max(1, min(max_slices_by_token_budget, max_slices_by_content)) |
| 1632 | |
| 1633 | def get_encoder_cudagraph_config(self) -> EncoderCudaGraphConfig: |
| 1634 | buffer_keys = [ |
| 1635 | _MINICPMV_CUDAGRAPH_BUF_KEY_PIXEL, |
| 1636 | _MINICPMV_CUDAGRAPH_BUF_KEY_TGT_SIZES, |
| 1637 | _MINICPMV_CUDAGRAPH_BUF_KEY_PATCH_MASK, |
| 1638 | ] |
| 1639 | # Video is only supported from 2.6 onward. |
| 1640 | modalities = ["image"] |
| 1641 | if self.version in {(2, 6), (4, 0)}: |
| 1642 | modalities.append("video") |
| 1643 | |
| 1644 | max_frames = self.get_max_frames_per_video() if "video" in modalities else 1 |
| 1645 | |
| 1646 | return EncoderCudaGraphConfig( |
| 1647 | modalities=modalities, |
| 1648 | buffer_keys=buffer_keys, |
| 1649 | out_hidden_size=int(self.embed_dim), |
| 1650 | max_frames_per_video=max_frames, |
| 1651 | capture_axes=(self._mcpmv_patch_grid_keys(),), |
| 1652 | ) |
| 1653 | |
| 1654 | def get_input_modality(self, mm_kwargs: dict[str, Any]) -> str: |
| 1655 | if "video_pixel_values" in mm_kwargs: |
| 1656 | return "video" |
| 1657 | return "image" |
| 1658 | |
| 1659 | def get_max_frames_per_video(self) -> int: |
| 1660 | info = MULTIMODAL_REGISTRY.get_processing_info(self.vllm_config.model_config) |
| 1661 | assert isinstance(info, MiniCPMVProcessingInfo) |
| 1662 | return int( |
| 1663 | info.get_num_frames_with_most_features( |
| 1664 | seq_len=self.vllm_config.model_config.max_model_len, |
| 1665 | mm_counts={ |
| 1666 | "video": self.multimodal_config.get_limit_per_prompt("video") |
| 1667 | }, |
| 1668 | ) |
| 1669 | ) |
| 1670 | |
| 1671 | def get_encoder_cudagraph_budget_range( |
| 1672 | self, vllm_config: VllmConfig |
| 1673 | ) -> tuple[int, int]: |
| 1674 | # Each slice produces exactly query_num resampler output tokens. |
| 1675 | # A thumbnail-only image has 1 slice, so query_num is the smallest |
| 1676 | # possible encoder output and the natural minimum budget. |
| 1677 | min_budget = int(self.config.query_num) |
| 1678 | max_budget = min( |
| 1679 | vllm_config.scheduler_config.max_num_batched_tokens, |
| 1680 | vllm_config.model_config.max_model_len, |
| 1681 | ) |
| 1682 | return (min_budget, max_budget) |
| 1683 | |
| 1684 | def get_encoder_cudagraph_item_specs( |
| 1685 | self, mm_kwargs: dict[str, Any] |
| 1686 | ) -> list[EncoderItemSpec]: |
| 1687 | video = self.get_input_modality(mm_kwargs) == "video" |
| 1688 | pixel_values_key = "video_pixel_values" if video else "pixel_values" |
| 1689 | pixel_values: list[list[torch.Tensor]] = mm_kwargs[pixel_values_key] |
| 1690 | slice_counts = [len(img) for img in pixel_values] |
| 1691 | tgt_sizes = _mcpmv_tgt_sizes_tensor(mm_kwargs, video=video) |
| 1692 | tgt_sizes = _mcpmv_normalize_tgt_sizes(tgt_sizes, slice_counts) |
| 1693 | patch_sums = tgt_sizes.prod(-1) |
| 1694 | input_sizes = [ |
| 1695 | int(group.sum().item()) for group in torch.split(patch_sums, slice_counts) |
| 1696 | ] |
| 1697 | query_num = int(self.config.query_num) |
| 1698 | return [ |
| 1699 | EncoderItemSpec( |
| 1700 | input_size=input_sizes[i], |
| 1701 | output_tokens=slice_counts[i] * query_num, |
| 1702 | ) |
| 1703 | for i in range(len(pixel_values)) |
| 1704 | ] |
| 1705 | |
| 1706 | def select_encoder_cudagraph_items( |
| 1707 | self, |
| 1708 | mm_kwargs: dict[str, Any], |
| 1709 | indices: list[int], |
| 1710 | ) -> dict[str, Any]: |
| 1711 | subset, patch_grid = self._mcpmv_select_items(mm_kwargs, indices) |
| 1712 | subset[ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG] = (patch_grid,) |
| 1713 | return subset |
| 1714 | |
| 1715 | def _mcpmv_select_items( |
| 1716 | self, mm_kwargs: dict[str, Any], indices: list[int] |
| 1717 | ) -> tuple[dict[str, Any], tuple[int, int]]: |
| 1718 | """Slice mm_kwargs for `indices`; also returns the patch-grid key.""" |
| 1719 | video = self.get_input_modality(mm_kwargs) == "video" |
| 1720 | pixel_values_key = "video_pixel_values" if video else "pixel_values" |
| 1721 | tgt_key = "video_tgt_sizes" if video else "tgt_sizes" |
| 1722 | flat_key = ( |
| 1723 | _MINICPMV_CUDAGRAPH_FLAT_KEY_VIDEO |
| 1724 | if video |
| 1725 | else _MINICPMV_CUDAGRAPH_FLAT_KEY_IMAGE |
| 1726 | ) |
| 1727 | device = next(self.vpm.parameters()).device |
| 1728 | pixel_values: list[list[torch.Tensor]] = mm_kwargs[pixel_values_key] |
| 1729 | tgt_sizes = _mcpmv_tgt_sizes_tensor(mm_kwargs, video=video) |
| 1730 | |
| 1731 | subset = { |
| 1732 | k: v |
| 1733 | for k, v in mm_kwargs.items() |
| 1734 | if k not in _ENCODER_CUDAGRAPH_MM_KWARGS_SKIP_KEYS |
| 1735 | } |
| 1736 | |
| 1737 | if not indices: |
| 1738 | pixel_h, pixel_w = self._mcpmv_slice_pixel_size() |
| 1739 | vpm_dtype = next(self.vpm.parameters()).dtype |
| 1740 | subset.update( |
| 1741 | { |
| 1742 | pixel_values_key: [], |
| 1743 | tgt_key: torch.zeros((0, 2), dtype=torch.long, device=device), |
| 1744 | flat_key: torch.zeros( |
| 1745 | (0, 3 * pixel_h * pixel_w), device=device, dtype=vpm_dtype |
| 1746 | ), |
| 1747 | } |
| 1748 | ) |
| 1749 | return subset, self._mcpmv_patch_grid_keys()[0] |
| 1750 | |
| 1751 | slice_counts = [len(item_slices) for item_slices in pixel_values] |
| 1752 | tgt_sizes = _mcpmv_normalize_tgt_sizes(tgt_sizes, slice_counts) |
| 1753 | tgt_groups = torch.split(tgt_sizes, slice_counts) |
| 1754 | |
| 1755 | patch_grid = self._mcpmv_resolve_patch_grid(tgt_sizes, slice_counts, indices) |
| 1756 | pixel_h, pixel_w = self._mcpmv_patch_grid_pixel_hw(patch_grid) |
| 1757 | |
| 1758 | selected_pixel_values = [pixel_values[i] for i in indices] |
| 1759 | selected_tgt_sizes_list = [tgt_groups[i] for i in indices] |
| 1760 | selected_tgt_sizes = torch.cat(selected_tgt_sizes_list, dim=0) |
| 1761 | |
| 1762 | selected_slices = flatten_2d_lists(selected_pixel_values) |
| 1763 | packed_flat_pixels = _mcpmv_pack_flat_pixels( |
| 1764 | selected_slices, |
| 1765 | pixel_height=pixel_h, |
| 1766 | pixel_width=pixel_w, |
| 1767 | max_num_slices=len(selected_slices), |
| 1768 | device=selected_slices[0].device, |
| 1769 | dtype=selected_slices[0].dtype, |
| 1770 | patch_size=int(self.vpm.embeddings.patch_size), |
| 1771 | tgt_sizes=selected_tgt_sizes, |
| 1772 | ) |
| 1773 | |
| 1774 | subset.update( |
| 1775 | { |
| 1776 | pixel_values_key: selected_pixel_values, |
| 1777 | tgt_key: selected_tgt_sizes_list, |
| 1778 | flat_key: packed_flat_pixels, |
| 1779 | _MINICPMV_CUDAGRAPH_PATCH_GRID_KEY: patch_grid, |
| 1780 | } |
| 1781 | ) |
| 1782 | return subset, patch_grid |
| 1783 | |
| 1784 | def prepare_encoder_cudagraph_capture_inputs( |
| 1785 | self, |
| 1786 | token_budget: int, |
| 1787 | max_batch_size: int, |
| 1788 | max_frames_per_batch: int, |
| 1789 | device: torch.device, |
| 1790 | dtype: torch.dtype, |
| 1791 | path: str = "default", |
| 1792 | axis_keys: tuple[Hashable, ...] | None = None, |
| 1793 | ): |
| 1794 | patch_grid = ( |
| 1795 | cast("tuple[int, int]", axis_keys[0]) |
| 1796 | if axis_keys |
| 1797 | # Without capture-axis context, use the largest (full-resolution) |
| 1798 | # patch grid. |
| 1799 | else self._mcpmv_patch_grid_keys()[-1] |
| 1800 | ) |
| 1801 | return self._mcpmv_capture_inputs( |
| 1802 | token_budget, |
| 1803 | max_batch_size, |
| 1804 | max_frames_per_batch, |
| 1805 | device, |
| 1806 | dtype, |
| 1807 | patch_grid=patch_grid, |
| 1808 | ) |
| 1809 | |
| 1810 | def _mcpmv_capture_inputs( |
| 1811 | self, |
| 1812 | token_budget: int, |
| 1813 | max_batch_size: int, |
| 1814 | max_frames_per_batch: int, |
| 1815 | device: torch.device, |
| 1816 | dtype: torch.dtype, |
| 1817 | patch_grid: tuple[int, int], |
| 1818 | ): |
| 1819 | th, tw = patch_grid |
| 1820 | pixel_h, pixel_w = self._mcpmv_patch_grid_pixel_hw(patch_grid) |
| 1821 | max_patches = self._mcpmv_patch_grid_num_patches(patch_grid) |
| 1822 | max_num_slices = self._mcpmv_max_slices_cap( |
| 1823 | token_budget, |
| 1824 | max_batch_size, |
| 1825 | max_frames_per_batch, |
| 1826 | ) |
| 1827 | pixel_buffer = torch.zeros( |
| 1828 | (max_num_slices, 3, pixel_h, pixel_w), device=device, dtype=dtype |
| 1829 | ) |
| 1830 | dummy_tgt_sizes = torch.zeros( |
| 1831 | (max_num_slices, 2), dtype=torch.long, device=device |
| 1832 | ) |
| 1833 | dummy_tgt_sizes[:, 0] = th |
| 1834 | dummy_tgt_sizes[:, 1] = tw |
| 1835 | dummy_patch_mask = torch.ones( |
| 1836 | (max_num_slices, max_patches), dtype=torch.bool, device=device |
| 1837 | ) |
| 1838 | values: dict[str, torch.Tensor] = { |
| 1839 | _MINICPMV_CUDAGRAPH_BUF_KEY_PIXEL: pixel_buffer, |
| 1840 | _MINICPMV_CUDAGRAPH_BUF_KEY_TGT_SIZES: dummy_tgt_sizes, |
| 1841 | _MINICPMV_CUDAGRAPH_BUF_KEY_PATCH_MASK: dummy_patch_mask, |
| 1842 | } |
| 1843 | return EncoderCudaGraphCaptureInputs(values=values) |
| 1844 | |
| 1845 | def prepare_encoder_cudagraph_replay_buffers( |
| 1846 | self, |
| 1847 | mm_kwargs: dict[str, Any], |
| 1848 | max_batch_size: int, |
| 1849 | max_frames_per_batch: int, |
| 1850 | path: str = "default", |
| 1851 | ): |
| 1852 | _ = max_batch_size |
| 1853 | _ = max_frames_per_batch |
| 1854 | video = self.get_input_modality(mm_kwargs) == "video" |
| 1855 | flat_key = ( |
| 1856 | _MINICPMV_CUDAGRAPH_FLAT_KEY_VIDEO |
| 1857 | if video |
| 1858 | else _MINICPMV_CUDAGRAPH_FLAT_KEY_IMAGE |
| 1859 | ) |
| 1860 | flat_pixels = mm_kwargs[flat_key] # (num_actual_slices, 3*pixel_h*pixel_w) |
| 1861 | patch_grid = mm_kwargs[_MINICPMV_CUDAGRAPH_PATCH_GRID_KEY] |
| 1862 | pixel_h, pixel_w = self._mcpmv_patch_grid_pixel_hw(patch_grid) |
| 1863 | max_patches = self._mcpmv_patch_grid_num_patches(patch_grid) |
| 1864 | pixel_buffer = flat_pixels.reshape(-1, 3, pixel_h, pixel_w) |
| 1865 | |
| 1866 | device = next(self.vpm.parameters()).device |
| 1867 | tgt_sizes_raw = _mcpmv_tgt_sizes_tensor(mm_kwargs, video=video) |
| 1868 | if isinstance(tgt_sizes_raw, list): |
| 1869 | tgt_sizes_raw = torch.cat(tgt_sizes_raw, dim=0) |
| 1870 | # tgt_sizes arrives from CPU-side mm_kwargs; use a pinned async copy |
| 1871 | # to stay clean under VLLM_GPU_SYNC_CHECK. |
| 1872 | tgt_sizes = async_tensor_h2d(tgt_sizes_raw, device, dtype=torch.long) |
| 1873 | |
| 1874 | patches_per_slice = tgt_sizes.prod(-1).clamp(max=max_patches) |
| 1875 | col_idx = torch.arange(max_patches, device=device) |
| 1876 | patch_attention_mask = col_idx.unsqueeze(0) < patches_per_slice.unsqueeze(1) |
| 1877 | |
| 1878 | values: dict[str, torch.Tensor] = { |
| 1879 | _MINICPMV_CUDAGRAPH_BUF_KEY_PIXEL: pixel_buffer, |
| 1880 | _MINICPMV_CUDAGRAPH_BUF_KEY_TGT_SIZES: tgt_sizes, |
| 1881 | _MINICPMV_CUDAGRAPH_BUF_KEY_PATCH_MASK: patch_attention_mask, |
| 1882 | } |
| 1883 | return EncoderCudaGraphReplayBuffers(values=values) |
| 1884 | |
| 1885 | def encoder_cudagraph_forward( |
| 1886 | self, |
| 1887 | values: dict[str, torch.Tensor], |
| 1888 | path: str = "default", |
| 1889 | ) -> torch.Tensor: |
| 1890 | all_pixel_values = values[_MINICPMV_CUDAGRAPH_BUF_KEY_PIXEL] |
| 1891 | tgt_sizes = values[_MINICPMV_CUDAGRAPH_BUF_KEY_TGT_SIZES] |
| 1892 | patch_mask = values[_MINICPMV_CUDAGRAPH_BUF_KEY_PATCH_MASK] |
| 1893 | patch_attention_mask = patch_mask.unsqueeze(1) |
| 1894 | |
| 1895 | max_num_slices = all_pixel_values.shape[0] |
| 1896 | # v2.5 infers patch layout from the attention mask; pass tgt_sizes=None. |
| 1897 | vpm_tgt_sizes = None if self.version == (2, 5) else tgt_sizes |
| 1898 | vision_embedding = self.vpm( |
| 1899 | all_pixel_values, |
| 1900 | patch_attention_mask=patch_attention_mask, |
| 1901 | tgt_sizes=vpm_tgt_sizes, |
| 1902 | ) |
| 1903 | |
| 1904 | resampler_out = self.resampler(vision_embedding, tgt_sizes) |
| 1905 | |
| 1906 | query_num = int(self.config.query_num) |
| 1907 | return resampler_out.reshape(max_num_slices * query_num, int(self.embed_dim)) |
| 1908 | |
| 1909 | def encoder_eager_forward( |
| 1910 | self, |
| 1911 | mm_kwargs: dict[str, Any], |
| 1912 | path: str = "default", |
| 1913 | ) -> torch.Tensor: |
| 1914 | """Eager encoder path; returns ``(total_tokens, embed_dim)`` like |
| 1915 | ``encoder_cudagraph_forward``. |
| 1916 | """ |
| 1917 | mm_kwargs_no_flat = { |
| 1918 | k: v |
| 1919 | for k, v in mm_kwargs.items() |
| 1920 | if k not in _ENCODER_CUDAGRAPH_MM_KWARGS_SKIP_KEYS |
| 1921 | } |
| 1922 | modalities = self._parse_and_validate_multimodal_inputs(**mm_kwargs_no_flat) |
| 1923 | segments: list[torch.Tensor] = [] |
| 1924 | embed_dim = self.embed_dim |
| 1925 | for modality in modalities: |
| 1926 | if modality == "images": |
| 1927 | image_input = modalities["images"] |
| 1928 | assert isinstance(image_input, MiniCPMVImagePixelInputs) |
| 1929 | image_embeddings = self.get_vision_hidden_states(image_input) |
| 1930 | segments.append(image_embeddings.reshape(-1, embed_dim)) |
| 1931 | elif modality == "videos": |
| 1932 | video_input = modalities["videos"] |
| 1933 | assert isinstance(video_input, MiniCPMVImagePixelInputs) |
| 1934 | video_embeddings = self.get_vision_hidden_states(video_input) |
| 1935 | segments.append(video_embeddings.reshape(-1, embed_dim)) |
| 1936 | if not segments: |
| 1937 | raise RuntimeError( |
| 1938 | "MiniCPM-V encoder cudagraph eager path expects pixel_values " |
| 1939 | "or video_pixel_values" |
| 1940 | ) |
| 1941 | return torch.cat(segments, dim=0) |
| 1942 | |
| 1943 | |
| 1944 | class MiniCPMV2_0(MiniCPMVBaseModel): |
| 1945 | supports_encoder_tp_data = False |
| 1946 | |
| 1947 | def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): |
| 1948 | super().__init__(vllm_config=vllm_config, prefix=prefix) |
| 1949 | assert self.version == (2, 0) |
| 1950 | |
| 1951 | def init_llm( |
| 1952 | self, |
| 1953 | vllm_config: VllmConfig, |
| 1954 | prefix: str = "", |
| 1955 | ) -> nn.Module: |
| 1956 | return MiniCPMForCausalLM(vllm_config=vllm_config, prefix=prefix) |
| 1957 | |
| 1958 | def init_vision_module( |
| 1959 | self, |
| 1960 | config: PreTrainedConfig, |
| 1961 | quant_config: QuantizationConfig | None, |
| 1962 | prefix: str = "", |
| 1963 | ) -> nn.Module: |
| 1964 | # TODO: refactor vision model through timm wrapper from transformers |
| 1965 | try: |
| 1966 | import timm |
| 1967 | except ImportError: |
| 1968 | raise ImportError("Please install timm==0.9.10") from ImportError |
| 1969 | |
| 1970 | with set_default_torch_dtype(torch.float16): |
| 1971 | model = timm.create_model( |
| 1972 | "vit_so400m_patch14_siglip_384.webli", |
| 1973 | pretrained=False, |
| 1974 | num_classes=0, |
| 1975 | dynamic_img_size=True, |
| 1976 | dynamic_img_pad=True, |
| 1977 | ) |
| 1978 | |
| 1979 | model = model.to(dtype=torch.get_default_dtype()) |
| 1980 | |
| 1981 | if ( |
| 1982 | isinstance(model, timm.models.VisionTransformer) |
| 1983 | and model.attn_pool is not None |
| 1984 | ): |
| 1985 | model.attn_pool = torch.nn.Identity() |
| 1986 | |
| 1987 | if self.config.drop_vision_last_layer: |
| 1988 | model.blocks = model.blocks[:-1] |
| 1989 | |
| 1990 | return model |
| 1991 | |
| 1992 | def init_resampler( |
| 1993 | self, |
| 1994 | embed_dim: int, |
| 1995 | vision_dim: int, |
| 1996 | quant_config: QuantizationConfig | None = None, |
| 1997 | prefix: str = "", |
| 1998 | ) -> nn.Module: |
| 1999 | with set_default_torch_dtype(torch.float16): |
| 2000 | resampler = Resampler2( |
| 2001 | embed_dim=embed_dim, |
| 2002 | num_heads=embed_dim // 128, |
| 2003 | grid_size=int(math.sqrt(self.config.query_num)), |
| 2004 | kv_dim=vision_dim, |
| 2005 | adaptive=False, |
| 2006 | do_post_projection=True, |
| 2007 | quant_config=quant_config, |
| 2008 | prefix=prefix, |
| 2009 | ) |
| 2010 | |
| 2011 | return resampler.to(dtype=torch.get_default_dtype()) |
| 2012 | |
| 2013 | def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor: |
| 2014 | pixel_values = data["pixel_values"] |
| 2015 | |
| 2016 | P_h, P_w = self.vpm.patch_embed.patch_size |
| 2017 | dtype: torch.dtype = self.vpm.pos_embed.data.dtype |
| 2018 | num_prefix_tokens = getattr(self.vpm, "num_prefix_tokens", 0) |
| 2019 | |
| 2020 | res = list[torch.Tensor]() |
| 2021 | for pixel_value in pixel_values: |
| 2022 | H, W = pixel_value[0].shape[-2:] |
| 2023 | tgt_size = (math.ceil(H / P_h), math.ceil(W / P_w)) |
| 2024 | vision_embedding = self.vpm.forward_features( |
| 2025 | pixel_value.unsqueeze(0).type(dtype) |
| 2026 | ) |
| 2027 | |
| 2028 | if num_prefix_tokens > 0: |
| 2029 | vision_embedding = vision_embedding[:, num_prefix_tokens:] |
| 2030 | res.append(self.resampler(vision_embedding, tgt_size)) |
| 2031 | |
| 2032 | return torch.vstack(res) |
| 2033 | |
| 2034 | |
| 2035 | class MiniCPMV2_5(_MiniCPMVEncoderCudaGraphMixin, SupportsLoRA): |
| 2036 | packed_modules_mapping = { |
| 2037 | "qkv_proj": [ |
| 2038 | "q_proj", |
| 2039 | "k_proj", |
| 2040 | "v_proj", |
| 2041 | ], |
| 2042 | "gate_up_proj": [ |
| 2043 | "gate_proj", |
| 2044 | "up_proj", |
| 2045 | ], |
| 2046 | } |
| 2047 | |
| 2048 | def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): |
| 2049 | super().__init__(vllm_config=vllm_config, prefix=prefix) |
| 2050 | assert self.version == (2, 5) |
| 2051 | |
| 2052 | def init_llm( |
| 2053 | self, |
| 2054 | vllm_config: VllmConfig, |
| 2055 | prefix: str = "", |
| 2056 | ) -> nn.Module: |
| 2057 | return LlamaForCausalLM(vllm_config=vllm_config, prefix=prefix) |
| 2058 | |
| 2059 | def init_vision_module( |
| 2060 | self, |
| 2061 | config: PreTrainedConfig, |
| 2062 | quant_config: QuantizationConfig | None, |
| 2063 | prefix: str = "", |
| 2064 | ) -> nn.Module: |
| 2065 | model = Idefics2VisionTransformer( |
| 2066 | config.vision_config, |
| 2067 | quant_config=quant_config, |
| 2068 | apply_encoder_attention_mask=True, |
| 2069 | prefix=prefix, |
| 2070 | ) |
| 2071 | if self.config.drop_vision_last_layer: |
| 2072 | model.encoder.layers = model.encoder.layers[:-1] |
| 2073 | return model |
| 2074 | |
| 2075 | def init_resampler( |
| 2076 | self, |
| 2077 | embed_dim: int, |
| 2078 | vision_dim: int, |
| 2079 | quant_config: QuantizationConfig | None = None, |
| 2080 | prefix: str = "", |
| 2081 | ) -> nn.Module: |
| 2082 | with set_default_torch_dtype(torch.float16): |
| 2083 | resampler = Resampler2_5( |
| 2084 | num_queries=self.config.query_num, |
| 2085 | embed_dim=embed_dim, |
| 2086 | num_heads=embed_dim // 128, |
| 2087 | kv_dim=vision_dim, |
| 2088 | quant_config=quant_config, |
| 2089 | prefix=prefix, |
| 2090 | ) |
| 2091 | |
| 2092 | return resampler.to(dtype=torch.get_default_dtype()) |
| 2093 | |
| 2094 | def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor: |
| 2095 | pixel_values = data["pixel_values"] |
| 2096 | tgt_sizes = data["tgt_sizes"] |
| 2097 | |
| 2098 | B = len(pixel_values) |
| 2099 | P = pixel_values[0].shape[-2] |
| 2100 | L = max(item.shape[-1] for item in pixel_values) |
| 2101 | device = pixel_values[0].device |
| 2102 | dtype = pixel_values[0].dtype |
| 2103 | |
| 2104 | all_pixel_values = torch.zeros((B, 3, P, L), dtype=dtype, device=device) |
| 2105 | for i, pixel_values_item in enumerate(pixel_values): |
| 2106 | L_item = pixel_values_item.shape[-1] |
| 2107 | all_pixel_values[i, ..., :L_item] = pixel_values_item |
| 2108 | |
| 2109 | num_patches = tgt_sizes.prod(-1) |
| 2110 | with gpu_sync_allowed(): |
| 2111 | max_patches = num_patches.max().item() |
| 2112 | assert isinstance(max_patches, int) |
| 2113 | |
| 2114 | patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device) |
| 2115 | for i, num_patches_item in enumerate(num_patches): |
| 2116 | patch_attn_mask[i, :num_patches_item] = True |
| 2117 | |
| 2118 | vision_embedding = self.vpm( |
| 2119 | all_pixel_values, |
| 2120 | patch_attention_mask=patch_attn_mask.unsqueeze(1), |
| 2121 | tgt_sizes=None, |
| 2122 | ) |
| 2123 | |
| 2124 | return self.resampler(vision_embedding, tgt_sizes) |
| 2125 | |
| 2126 | |
| 2127 | class MiniCPMV2_6(_MiniCPMVEncoderCudaGraphMixin, SupportsLoRA): |
| 2128 | hf_to_vllm_mapper = WeightsMapper( |
| 2129 | # The vision-only models have no audio tower or TTS head to load weights into. |
| 2130 | orig_to_new_prefix={"apm.": None, "audio": None, "tts": None} |
| 2131 | ) |
| 2132 | packed_modules_mapping = { |
| 2133 | "qkv_proj": [ |
| 2134 | "q_proj", |
| 2135 | "k_proj", |
| 2136 | "v_proj", |
| 2137 | ], |
| 2138 | "gate_up_proj": [ |
| 2139 | "gate_proj", |
| 2140 | "up_proj", |
| 2141 | ], |
| 2142 | } |
| 2143 | |
| 2144 | def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): |
| 2145 | super().__init__(vllm_config=vllm_config, prefix=prefix) |
| 2146 | assert self.version == (2, 6) |
| 2147 | |
| 2148 | def init_llm( |
| 2149 | self, |
| 2150 | vllm_config: VllmConfig, |
| 2151 | prefix: str = "", |
| 2152 | ) -> nn.Module: |
| 2153 | return Qwen2ForCausalLM(vllm_config=vllm_config, prefix=prefix) |
| 2154 | |
| 2155 | def init_vision_module( |
| 2156 | self, |
| 2157 | config: PreTrainedConfig, |
| 2158 | quant_config: QuantizationConfig | None = None, |
| 2159 | prefix: str = "", |
| 2160 | ) -> nn.Module: |
| 2161 | model = Idefics2VisionTransformer( |
| 2162 | config.vision_config, |
| 2163 | quant_config=quant_config, |
| 2164 | apply_encoder_attention_mask=True, |
| 2165 | prefix=prefix, |
| 2166 | ) |
| 2167 | if self.config.drop_vision_last_layer: |
| 2168 | model.encoder.layers = model.encoder.layers[:-1] |
| 2169 | return model |
| 2170 | |
| 2171 | def init_resampler( |
| 2172 | self, |
| 2173 | embed_dim: int, |
| 2174 | vision_dim: int, |
| 2175 | quant_config: QuantizationConfig | None = None, |
| 2176 | prefix: str = "", |
| 2177 | ) -> nn.Module: |
| 2178 | with set_default_torch_dtype(torch.float16): |
| 2179 | # The resampler in 2.6 remains consistent with the one in 2.5. |
| 2180 | resampler = Resampler2_5( |
| 2181 | num_queries=self.config.query_num, |
| 2182 | embed_dim=embed_dim, |
| 2183 | num_heads=embed_dim // 128, |
| 2184 | kv_dim=vision_dim, |
| 2185 | quant_config=quant_config, |
| 2186 | prefix=prefix, |
| 2187 | ) |
| 2188 | |
| 2189 | return resampler.to(dtype=torch.get_default_dtype()) |
| 2190 | |
| 2191 | def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor: |
| 2192 | pixel_values = data["pixel_values"] |
| 2193 | tgt_sizes = data["tgt_sizes"] |
| 2194 | |
| 2195 | B = len(pixel_values) |
| 2196 | P = pixel_values[0].shape[-2] |
| 2197 | L = max(item.shape[-1] for item in pixel_values) |
| 2198 | device = pixel_values[0].device |
| 2199 | dtype = pixel_values[0].dtype |
| 2200 | |
| 2201 | all_pixel_values = torch.zeros((B, 3, P, L), dtype=dtype, device=device) |
| 2202 | for i, pixel_values_item in enumerate(pixel_values): |
| 2203 | L_item = pixel_values_item.shape[-1] |
| 2204 | all_pixel_values[i, ..., :L_item] = pixel_values_item |
| 2205 | |
| 2206 | num_patches = tgt_sizes.prod(-1) |
| 2207 | # Needed as a Python int to size the mask below. |
| 2208 | with gpu_sync_allowed(): |
| 2209 | max_patches = num_patches.max().item() |
| 2210 | assert isinstance(max_patches, int) |
| 2211 | |
| 2212 | patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device) |
| 2213 | for i, num_patches_item in enumerate(num_patches): |
| 2214 | patch_attn_mask[i, :num_patches_item] = True |
| 2215 | |
| 2216 | vision_embedding = self.vpm( |
| 2217 | all_pixel_values, |
| 2218 | patch_attention_mask=patch_attn_mask.unsqueeze(1), |
| 2219 | tgt_sizes=tgt_sizes, |
| 2220 | ) |
| 2221 | |
| 2222 | return self.resampler(vision_embedding, tgt_sizes) |
| 2223 | |
| 2224 | def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: |
| 2225 | loader = AutoWeightsLoader(self) |
| 2226 | loaded = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) |
| 2227 | self._ensure_resampler_device() |
| 2228 | return loaded |
| 2229 | |
| 2230 | |
| 2231 | class MiniCPMV4_0(_MiniCPMVEncoderCudaGraphMixin, SupportsLoRA): |
| 2232 | hf_to_vllm_mapper = MiniCPMV2_6.hf_to_vllm_mapper |
| 2233 | packed_modules_mapping = { |
| 2234 | "qkv_proj": [ |
| 2235 | "q_proj", |
| 2236 | "k_proj", |
| 2237 | "v_proj", |
| 2238 | ], |
| 2239 | "gate_up_proj": [ |
| 2240 | "gate_proj", |
| 2241 | "up_proj", |
| 2242 | ], |
| 2243 | } |
| 2244 | |
| 2245 | def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): |
| 2246 | super().__init__(vllm_config=vllm_config, prefix=prefix) |
| 2247 | assert self.version == (4, 0) |
| 2248 | |
| 2249 | def init_llm( |
| 2250 | self, |
| 2251 | vllm_config: VllmConfig, |
| 2252 | prefix: str = "", |
| 2253 | ) -> nn.Module: |
| 2254 | return LlamaForCausalLM(vllm_config=vllm_config, prefix=prefix) |
| 2255 | |
| 2256 | def init_vision_module( |
| 2257 | self, |
| 2258 | config: PreTrainedConfig, |
| 2259 | quant_config: QuantizationConfig | None = None, |
| 2260 | prefix: str = "", |
| 2261 | ) -> nn.Module: |
| 2262 | model = Idefics2VisionTransformer( |
| 2263 | config.vision_config, |
| 2264 | quant_config=quant_config, |
| 2265 | apply_encoder_attention_mask=True, |
| 2266 | prefix=prefix, |
| 2267 | ) |
| 2268 | if self.config.drop_vision_last_layer: |
| 2269 | model.encoder.layers = model.encoder.layers[:-1] |
| 2270 | return model |
| 2271 | |
| 2272 | def init_resampler( |
| 2273 | self, |
| 2274 | embed_dim: int, |
| 2275 | vision_dim: int, |
| 2276 | quant_config: QuantizationConfig | None = None, |
| 2277 | prefix: str = "", |
| 2278 | ) -> nn.Module: |
| 2279 | with set_default_torch_dtype(torch.float16): |
| 2280 | # The resampler in 4.0 remains consistent with the one in 2.5/2.6. |
| 2281 | resampler = Resampler2_5( |
| 2282 | num_queries=self.config.query_num, |
| 2283 | embed_dim=embed_dim, |
| 2284 | num_heads=embed_dim // 128, |
| 2285 | kv_dim=vision_dim, |
| 2286 | quant_config=quant_config, |
| 2287 | prefix=prefix, |
| 2288 | ) |
| 2289 | return resampler.to(dtype=torch.get_default_dtype()) |
| 2290 | |
| 2291 | def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor: |
| 2292 | pixel_values = data["pixel_values"] |
| 2293 | tgt_sizes = data["tgt_sizes"] |
| 2294 | |
| 2295 | B = len(pixel_values) |
| 2296 | P = pixel_values[0].shape[-2] |
| 2297 | L = max(item.shape[-1] for item in pixel_values) |
| 2298 | device = pixel_values[0].device |
| 2299 | dtype = pixel_values[0].dtype |
| 2300 | |
| 2301 | all_pixel_values = torch.zeros((B, 3, P, L), dtype=dtype, device=device) |
| 2302 | for i, pixel_values_item in enumerate(pixel_values): |
| 2303 | L_item = pixel_values_item.shape[-1] |
| 2304 | all_pixel_values[i, ..., :L_item] = pixel_values_item |
| 2305 | |
| 2306 | num_patches = tgt_sizes.prod(-1) |
| 2307 | with gpu_sync_allowed(): |
| 2308 | max_patches = num_patches.max().item() |
| 2309 | assert isinstance(max_patches, int) |
| 2310 | |
| 2311 | patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device) |
| 2312 | for i, num_patches_item in enumerate(num_patches): |
| 2313 | patch_attn_mask[i, :num_patches_item] = True |
| 2314 | |
| 2315 | vision_embedding = self.vpm( |
| 2316 | all_pixel_values, |
| 2317 | patch_attention_mask=patch_attn_mask.unsqueeze(1), |
| 2318 | tgt_sizes=tgt_sizes, |
| 2319 | ) |
| 2320 | |
| 2321 | return self.resampler(vision_embedding, tgt_sizes) |
| 2322 | |
| 2323 | def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: |
| 2324 | loader = AutoWeightsLoader(self) |
| 2325 | loaded = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) |
| 2326 | self._ensure_resampler_device() |
| 2327 | return loaded |
| 2328 | |
| 2329 | |
| 2330 | class MiniCPMV4_5(MiniCPMVBaseModel, SupportsLoRA): |
| 2331 | hf_to_vllm_mapper = MiniCPMV2_6.hf_to_vllm_mapper |
| 2332 | packed_modules_mapping = { |
| 2333 | "qkv_proj": [ |
| 2334 | "q_proj", |
| 2335 | "k_proj", |
| 2336 | "v_proj", |
| 2337 | ], |
| 2338 | "gate_up_proj": [ |
| 2339 | "gate_proj", |
| 2340 | "up_proj", |
| 2341 | ], |
| 2342 | } |
| 2343 | |
| 2344 | def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): |
| 2345 | super().__init__(vllm_config=vllm_config, prefix=prefix) |
| 2346 | assert self.version == (4, 5) |
| 2347 | |
| 2348 | def init_llm( |
| 2349 | self, |
| 2350 | vllm_config: VllmConfig, |
| 2351 | prefix: str = "", |
| 2352 | ) -> nn.Module: |
| 2353 | return Qwen3ForCausalLM(vllm_config=vllm_config, prefix=prefix) |
| 2354 | |
| 2355 | def init_vision_module( |
| 2356 | self, |
| 2357 | config: PreTrainedConfig, |
| 2358 | quant_config: QuantizationConfig | None = None, |
| 2359 | prefix: str = "", |
| 2360 | ) -> nn.Module: |
| 2361 | model = Idefics2VisionTransformer( |
| 2362 | config.vision_config, |
| 2363 | quant_config=quant_config, |
| 2364 | apply_encoder_attention_mask=True, |
| 2365 | prefix=prefix, |
| 2366 | ) |
| 2367 | if self.config.drop_vision_last_layer: |
| 2368 | model.encoder.layers = model.encoder.layers[:-1] |
| 2369 | return model |
| 2370 | |
| 2371 | def init_resampler( |
| 2372 | self, |
| 2373 | embed_dim: int, |
| 2374 | vision_dim: int, |
| 2375 | quant_config: QuantizationConfig | None = None, |
| 2376 | prefix: str = "", |
| 2377 | ) -> nn.Module: |
| 2378 | with set_default_torch_dtype(torch.float16): |
| 2379 | # The resampler in 4.0 remains consistent with the one in 2.5/2.6. |
| 2380 | resampler = Resampler4_5( |
| 2381 | num_queries=self.config.query_num, |
| 2382 | embed_dim=embed_dim, |
| 2383 | num_heads=embed_dim // 128, |
| 2384 | kv_dim=vision_dim, |
| 2385 | quant_config=quant_config, |
| 2386 | prefix=prefix, |
| 2387 | ) |
| 2388 | |
| 2389 | return resampler.to(dtype=torch.get_default_dtype()) |
| 2390 | |
| 2391 | def get_vision_hidden_states(self, data: MiniCPMVImagePixelInputs) -> torch.Tensor: |
| 2392 | pixel_values = data["pixel_values"] |
| 2393 | tgt_sizes = data["tgt_sizes"] |
| 2394 | temporal_ids = data.get("temporal_ids", None) |
| 2395 | |
| 2396 | B = len(pixel_values) |
| 2397 | P = pixel_values[0].shape[-2] |
| 2398 | L = max(item.shape[-1] for item in pixel_values) |
| 2399 | device = pixel_values[0].device |
| 2400 | dtype = pixel_values[0].dtype |
| 2401 | |
| 2402 | all_pixel_values = torch.zeros((B, 3, P, L), dtype=dtype, device=device) |
| 2403 | all_temporal_ids = ( |
| 2404 | None if temporal_ids is None else flatten_2d_lists(temporal_ids) |
| 2405 | ) |
| 2406 | for i, pixel_values_item in enumerate(pixel_values): |
| 2407 | L_item = pixel_values_item.shape[-1] |
| 2408 | all_pixel_values[i, ..., :L_item] = pixel_values_item |
| 2409 | |
| 2410 | num_patches = tgt_sizes.prod(-1) |
| 2411 | with gpu_sync_allowed(): |
| 2412 | max_patches = num_patches.max().item() |
| 2413 | assert isinstance(max_patches, int) |
| 2414 | |
| 2415 | patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device) |
| 2416 | for i, num_patches_item in enumerate(num_patches): |
| 2417 | patch_attn_mask[i, :num_patches_item] = True |
| 2418 | |
| 2419 | vision_embedding = self.vpm( |
| 2420 | all_pixel_values, |
| 2421 | patch_attention_mask=patch_attn_mask.unsqueeze(1), |
| 2422 | tgt_sizes=tgt_sizes, |
| 2423 | ) |
| 2424 | |
| 2425 | return self.resampler(vision_embedding, tgt_sizes, all_temporal_ids) |
| 2426 | |
| 2427 | def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: |
| 2428 | loader = AutoWeightsLoader(self) |
| 2429 | loaded = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) |
| 2430 | self._ensure_resampler_device() |
| 2431 | return loaded |
| 2432 | |
| 2433 | |
| 2434 | _SUPPORT_VERSION = { |
| 2435 | (2, 0): MiniCPMV2_0, |
| 2436 | (2, 5): MiniCPMV2_5, |
| 2437 | (2, 6): MiniCPMV2_6, |
| 2438 | (4, 0): MiniCPMV4_0, |
| 2439 | (4, 5): MiniCPMV4_5, |
| 2440 | } |
| 2441 | |
| 2442 | |
| 2443 | @MULTIMODAL_REGISTRY.register_processor( |
| 2444 | MiniCPMVMultiModalProcessor, |
| 2445 | info=MiniCPMVProcessingInfo, |
| 2446 | dummy_inputs=MiniCPMVDummyInputsBuilder, |
| 2447 | ) |
| 2448 | class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA): |
| 2449 | """Different versions of MiniCPMV use different visual encoders and LLMs, |
| 2450 | which is not conducive to the current integration logic of LoRA and |
| 2451 | bitsandbytes in vLLM. Therefore, it is necessary to separate them. |
| 2452 | """ |
| 2453 | |
| 2454 | def __new__(cls, *, vllm_config: VllmConfig, prefix: str = ""): |
| 2455 | config = vllm_config.model_config.hf_config |
| 2456 | version_values: tuple[int, ...] |
| 2457 | if not hasattr(config, "version"): |
| 2458 | if config.hidden_size == 2304 and config.query_num == 64: |
| 2459 | version_values = (2, 0) |
| 2460 | else: |
| 2461 | version_values = (2, 5) |
| 2462 | else: |
| 2463 | version_values = tuple(int(x) for x in str(config.version).split(".")) |
| 2464 | # Dispatch class based on version |
| 2465 | if len(version_values) == 2: |
| 2466 | version_key = (version_values[0], version_values[1]) |
| 2467 | instance_cls = _SUPPORT_VERSION.get(version_key) |
| 2468 | else: |
| 2469 | instance_cls = None |
| 2470 | if instance_cls is None: |
| 2471 | supported_versions = ", ".join( |
| 2472 | [f"{v[0]}.{v[1]}" for v in sorted(_SUPPORT_VERSION.keys())] |
| 2473 | ) |
| 2474 | raise ValueError( |
| 2475 | f"Currently, MiniCPMV only supports versions " |
| 2476 | f"{supported_versions}. Got version: {version_values}" |
| 2477 | ) |
| 2478 | |
| 2479 | # quant_config references base class members, |
| 2480 | # so update values before init is called |
| 2481 | cls.packed_modules_mapping.update(instance_cls.packed_modules_mapping) |
| 2482 | cls.embedding_modules.update(instance_cls.embedding_modules) |
| 2483 | return instance_cls(vllm_config=vllm_config, prefix=prefix) |