| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """MiniMax M3 VL HuggingFace-compatible Processor / ImageProcessor / |
| 4 | VideoProcessor, vendored into vLLM so the model loads without |
| 5 | ``--trust-remote-code`` (the released checkpoint only ships these classes as |
| 6 | remote code via ``auto_map``). |
| 7 | |
| 8 | Adapted verbatim from the ``MiniMaxAI/Minimax-M3-preview`` repository files |
| 9 | ``image_processor.py``, ``video_processor.py`` and ``processing_minimax.py`` |
| 10 | (revision ``db01c0fe``). Both image and video processors use Qwen-style |
| 11 | ``smart_resize`` (bound by total pixels). The original async frame-sampling |
| 12 | helpers are intentionally omitted: vLLM performs its own frame loading and |
| 13 | feeds decoded frames to the processor. |
| 14 | """ |
| 15 | |
| 16 | import math |
| 17 | |
| 18 | import torch |
| 19 | from torchvision.transforms import InterpolationMode |
| 20 | from transformers import AutoTokenizer, BatchFeature |
| 21 | from transformers.image_processing_backends import ( |
| 22 | TorchvisionBackend, |
| 23 | group_images_by_shape, |
| 24 | reorder_images, |
| 25 | ) |
| 26 | from transformers.image_utils import PILImageResampling, SizeDict |
| 27 | from transformers.processing_utils import ( |
| 28 | ImagesKwargs, |
| 29 | ProcessingKwargs, |
| 30 | ProcessorMixin, |
| 31 | Unpack, |
| 32 | VideosKwargs, |
| 33 | ) |
| 34 | from transformers.utils import TensorType |
| 35 | from transformers.video_processing_utils import BaseVideoProcessor |
| 36 | from transformers.video_utils import group_videos_by_shape, reorder_videos |
| 37 | |
| 38 | # Maximum allowed aspect ratio before smart_resize rejects the input. |
| 39 | MAX_RATIO = 200 |
| 40 | |
| 41 | # Fixed (non-configurable) bounds for the long-side resize logic, per the |
| 42 | # MiniMax-M3 size spec. ``min_short_side_pixel`` is the floor the short edge is |
| 43 | # enlarged to; ``*_MAX_TOTAL_PIXELS`` is the hard area cap that, once exceeded, |
| 44 | # aborts processing instead of downscaling. |
| 45 | MIN_SHORT_SIDE_PIXEL = 112 |
| 46 | IMAGE_MAX_TOTAL_PIXELS = 12_845_056 # 3584 ** 2 (width * height) |
| 47 | VIDEO_MAX_TOTAL_PIXELS = 301_056_000 # width * height * frames |
| 48 | |
| 49 | |
| 50 | def round_by_factor(number: int | float, factor: int) -> int: |
| 51 | return round(number / factor) * factor |
| 52 | |
| 53 | |
| 54 | def ceil_by_factor(number: int | float, factor: int) -> int: |
| 55 | return math.ceil(number / factor) * factor |
| 56 | |
| 57 | |
| 58 | def floor_by_factor(number: int | float, factor: int) -> int: |
| 59 | return math.floor(number / factor) * factor |
| 60 | |
| 61 | |
| 62 | def _smart_resize_by_long_side( |
| 63 | height: int, |
| 64 | width: int, |
| 65 | factor: int, |
| 66 | max_long_side_pixel: int, |
| 67 | min_short_side_pixel: int, |
| 68 | max_total_pixels: int | None, |
| 69 | ) -> tuple[int, int]: |
| 70 | """Long-side based resize (MiniMax-M3 size spec). |
| 71 | |
| 72 | (a) if the long side exceeds ``max_long_side_pixel`` → shrink so the long |
| 73 | side equals ``max_long_side_pixel``; |
| 74 | (b) else if the short side is below ``min_short_side_pixel`` → enlarge so the |
| 75 | short side equals ``min_short_side_pixel``; |
| 76 | (c) if the resulting area still exceeds ``max_total_pixels`` → raise. |
| 77 | |
| 78 | (a) and (b) are mutually exclusive (they branch on the *original* long side). |
| 79 | Both sides are then rounded to a multiple of ``factor``. For videos the |
| 80 | ``max_total_pixels`` cap is volumetric (width * height * frames) and is |
| 81 | enforced by the caller, so pass ``max_total_pixels=None`` here. |
| 82 | """ |
| 83 | long_side = max(height, width) |
| 84 | short_side = min(height, width) |
| 85 | |
| 86 | scaled_height: float = height |
| 87 | scaled_width: float = width |
| 88 | if long_side > max_long_side_pixel: |
| 89 | beta = max_long_side_pixel / long_side |
| 90 | scaled_height = height * beta |
| 91 | scaled_width = width * beta |
| 92 | elif short_side < min_short_side_pixel: |
| 93 | beta = min_short_side_pixel / short_side |
| 94 | scaled_height = height * beta |
| 95 | scaled_width = width * beta |
| 96 | |
| 97 | h_bar = max(factor, round_by_factor(scaled_height, factor)) |
| 98 | w_bar = max(factor, round_by_factor(scaled_width, factor)) |
| 99 | |
| 100 | if max_total_pixels is not None and h_bar * w_bar > max_total_pixels: |
| 101 | raise ValueError( |
| 102 | f"image area {h_bar * w_bar} exceeds max_total_pixels " |
| 103 | f"{max_total_pixels} after resizing" |
| 104 | ) |
| 105 | return h_bar, w_bar |
| 106 | |
| 107 | |
| 108 | def smart_resize( |
| 109 | height: int, |
| 110 | width: int, |
| 111 | factor: int = 28, |
| 112 | min_pixels: int = 4 * 28 * 28, |
| 113 | max_pixels: int = 451584, |
| 114 | max_long_side_pixel: int | None = None, |
| 115 | min_short_side_pixel: int = MIN_SHORT_SIDE_PIXEL, |
| 116 | max_total_pixels: int | None = None, |
| 117 | ) -> tuple[int, int]: |
| 118 | """Rescale (height, width) so each side is a multiple of ``factor``. |
| 119 | |
| 120 | When ``max_long_side_pixel`` is set, use the MiniMax-M3 long-side resize |
| 121 | spec (see :func:`_smart_resize_by_long_side`). Otherwise fall back to the |
| 122 | Qwen-VL area bound, keeping the total area within ``[min_pixels, max_pixels]``. |
| 123 | """ |
| 124 | if max(height, width) / min(height, width) > MAX_RATIO: |
| 125 | raise ValueError( |
| 126 | f"absolute aspect ratio must be smaller than {MAX_RATIO}, " |
| 127 | f"got {max(height, width) / min(height, width)}" |
| 128 | ) |
| 129 | if max_long_side_pixel is not None: |
| 130 | return _smart_resize_by_long_side( |
| 131 | height, |
| 132 | width, |
| 133 | factor=factor, |
| 134 | max_long_side_pixel=max_long_side_pixel, |
| 135 | min_short_side_pixel=min_short_side_pixel, |
| 136 | max_total_pixels=max_total_pixels, |
| 137 | ) |
| 138 | h_bar = max(factor, round_by_factor(height, factor)) |
| 139 | w_bar = max(factor, round_by_factor(width, factor)) |
| 140 | if h_bar * w_bar > max_pixels: |
| 141 | beta = math.sqrt((height * width) / max_pixels) |
| 142 | h_bar = floor_by_factor(height / beta, factor) |
| 143 | w_bar = floor_by_factor(width / beta, factor) |
| 144 | elif h_bar * w_bar < min_pixels: |
| 145 | beta = math.sqrt(min_pixels / (height * width)) |
| 146 | h_bar = ceil_by_factor(height * beta, factor) |
| 147 | w_bar = ceil_by_factor(width * beta, factor) |
| 148 | return h_bar, w_bar |
| 149 | |
| 150 | |
| 151 | class MiniMaxM3VLImageProcessorKwargs(ImagesKwargs, total=False): # type: ignore[call-arg] |
| 152 | patch_size: int |
| 153 | temporal_patch_size: int |
| 154 | merge_size: int |
| 155 | max_pixels: int |
| 156 | max_long_side_pixel: int |
| 157 | |
| 158 | |
| 159 | class MiniMaxM3VLImageProcessor(TorchvisionBackend): |
| 160 | do_resize = True |
| 161 | resample = PILImageResampling.BICUBIC |
| 162 | # required by base-class validation, not used as the resize bound |
| 163 | size = {"height": 672, "width": 672} |
| 164 | default_to_square = False |
| 165 | do_rescale = True |
| 166 | rescale_factor = 1 / 255 |
| 167 | do_normalize = True |
| 168 | image_mean = [0.48145466, 0.4578275, 0.40821073] |
| 169 | image_std = [0.26862954, 0.26130258, 0.27577711] |
| 170 | do_convert_rgb = True |
| 171 | patch_size = 14 |
| 172 | temporal_patch_size = 2 |
| 173 | merge_size = 2 |
| 174 | max_pixels = 451584 # 672 * 672 |
| 175 | # Long-side resize spec (opt-in via ``max_long_side_pixel``). The latter two |
| 176 | # are fixed per the spec and are not exposed as configurable kwargs. |
| 177 | max_long_side_pixel = None |
| 178 | min_short_side_pixel = MIN_SHORT_SIDE_PIXEL |
| 179 | max_total_pixels = IMAGE_MAX_TOTAL_PIXELS |
| 180 | valid_kwargs = MiniMaxM3VLImageProcessorKwargs |
| 181 | model_input_names = ["pixel_values", "image_grid_thw"] |
| 182 | |
| 183 | def __init__(self, **kwargs: Unpack[MiniMaxM3VLImageProcessorKwargs]): |
| 184 | super().__init__(**kwargs) |
| 185 | |
| 186 | def preprocess( |
| 187 | self, images, **kwargs: Unpack[MiniMaxM3VLImageProcessorKwargs] |
| 188 | ) -> BatchFeature: |
| 189 | return super().preprocess(images, **kwargs) |
| 190 | |
| 191 | def _preprocess( |
| 192 | self, |
| 193 | images: list[torch.Tensor], |
| 194 | do_resize: bool, |
| 195 | size: SizeDict, |
| 196 | resample: "PILImageResampling | InterpolationMode | int | None", |
| 197 | do_rescale: bool, |
| 198 | rescale_factor: float, |
| 199 | do_normalize: bool, |
| 200 | image_mean: "float | list[float] | None", |
| 201 | image_std: "float | list[float] | None", |
| 202 | patch_size: int, |
| 203 | temporal_patch_size: int, |
| 204 | merge_size: int, |
| 205 | max_pixels: int, |
| 206 | max_long_side_pixel: "int | None", |
| 207 | disable_grouping: "bool | None", |
| 208 | return_tensors: "str | TensorType | None", |
| 209 | **kwargs, |
| 210 | ) -> BatchFeature: |
| 211 | grouped_images, grouped_images_index = group_images_by_shape( |
| 212 | images, disable_grouping=disable_grouping |
| 213 | ) |
| 214 | resized_images_grouped = {} |
| 215 | factor = patch_size * merge_size |
| 216 | for shape, stacked_images in grouped_images.items(): |
| 217 | height, width = stacked_images.shape[-2:] |
| 218 | if do_resize: |
| 219 | resized_height, resized_width = smart_resize( |
| 220 | height, |
| 221 | width, |
| 222 | factor=factor, |
| 223 | max_pixels=max_pixels, |
| 224 | max_long_side_pixel=max_long_side_pixel, |
| 225 | min_short_side_pixel=self.min_short_side_pixel, |
| 226 | max_total_pixels=self.max_total_pixels, |
| 227 | ) |
| 228 | stacked_images = self.resize( |
| 229 | stacked_images, |
| 230 | size=SizeDict(height=resized_height, width=resized_width), |
| 231 | resample=resample, |
| 232 | ) |
| 233 | resized_images_grouped[shape] = stacked_images |
| 234 | |
| 235 | resized_images = reorder_images(resized_images_grouped, grouped_images_index) |
| 236 | |
| 237 | grouped_images, grouped_images_index = group_images_by_shape( |
| 238 | resized_images, disable_grouping=disable_grouping |
| 239 | ) |
| 240 | processed_images_grouped = {} |
| 241 | processed_grids = {} |
| 242 | |
| 243 | for shape, stacked_images in grouped_images.items(): |
| 244 | resized_height, resized_width = stacked_images.shape[-2:] |
| 245 | |
| 246 | patches = self.rescale_and_normalize( |
| 247 | stacked_images, |
| 248 | do_rescale, |
| 249 | rescale_factor, |
| 250 | do_normalize, |
| 251 | image_mean, |
| 252 | image_std, |
| 253 | ) |
| 254 | if patches.ndim == 4: |
| 255 | patches = patches.unsqueeze(1) |
| 256 | |
| 257 | if patches.shape[1] % temporal_patch_size != 0: |
| 258 | repeats = patches[:, -1:].repeat( |
| 259 | 1, |
| 260 | temporal_patch_size - (patches.shape[1] % temporal_patch_size), |
| 261 | 1, |
| 262 | 1, |
| 263 | 1, |
| 264 | ) |
| 265 | patches = torch.cat([patches, repeats], dim=1) |
| 266 | |
| 267 | batch_size, grid_t, channel = patches.shape[:3] |
| 268 | grid_t = grid_t // temporal_patch_size |
| 269 | grid_h, grid_w = resized_height // patch_size, resized_width // patch_size |
| 270 | |
| 271 | patches = patches.view( |
| 272 | batch_size, |
| 273 | grid_t, |
| 274 | temporal_patch_size, |
| 275 | channel, |
| 276 | grid_h // merge_size, |
| 277 | merge_size, |
| 278 | patch_size, |
| 279 | grid_w // merge_size, |
| 280 | merge_size, |
| 281 | patch_size, |
| 282 | ) |
| 283 | patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9) |
| 284 | |
| 285 | flatten_patches = patches.reshape( |
| 286 | batch_size, |
| 287 | grid_t * grid_h * grid_w, |
| 288 | channel * temporal_patch_size * patch_size * patch_size, |
| 289 | ) |
| 290 | |
| 291 | processed_images_grouped[shape] = flatten_patches |
| 292 | processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size |
| 293 | |
| 294 | processed_images = reorder_images( |
| 295 | processed_images_grouped, grouped_images_index |
| 296 | ) |
| 297 | processed_grids = reorder_images(processed_grids, grouped_images_index) |
| 298 | |
| 299 | pixel_values = torch.cat(processed_images, dim=0) |
| 300 | image_grid_thw = torch.tensor(processed_grids, dtype=torch.long) |
| 301 | |
| 302 | return BatchFeature( |
| 303 | data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}, |
| 304 | tensor_type=return_tensors, |
| 305 | ) |
| 306 | |
| 307 | def get_number_of_image_patches(self, height: int, width: int, images_kwargs=None): |
| 308 | images_kwargs = images_kwargs or {} |
| 309 | patch_size = images_kwargs.get("patch_size", self.patch_size) |
| 310 | merge_size = images_kwargs.get("merge_size", self.merge_size) |
| 311 | max_pixels = images_kwargs.get("max_pixels", self.max_pixels) |
| 312 | max_long_side_pixel = images_kwargs.get( |
| 313 | "max_long_side_pixel", self.max_long_side_pixel |
| 314 | ) |
| 315 | |
| 316 | resized_height, resized_width = smart_resize( |
| 317 | height, |
| 318 | width, |
| 319 | factor=patch_size * merge_size, |
| 320 | max_pixels=max_pixels, |
| 321 | max_long_side_pixel=max_long_side_pixel, |
| 322 | min_short_side_pixel=self.min_short_side_pixel, |
| 323 | max_total_pixels=self.max_total_pixels, |
| 324 | ) |
| 325 | grid_h, grid_w = resized_height // patch_size, resized_width // patch_size |
| 326 | return grid_h * grid_w |
| 327 | |
| 328 | |
| 329 | class MiniMaxM3VLVideoProcessorKwargs(VideosKwargs, total=False): # type: ignore[call-arg] |
| 330 | patch_size: int |
| 331 | temporal_patch_size: int |
| 332 | merge_size: int |
| 333 | min_pixels: int |
| 334 | max_pixels: int |
| 335 | max_long_side_pixel: int |
| 336 | total_pixels: int |
| 337 | min_frames: int |
| 338 | max_frames: int |
| 339 | fps: "float | int" |
| 340 | |
| 341 | |
| 342 | class MiniMaxM3VLVideoProcessor(BaseVideoProcessor): |
| 343 | do_resize = True |
| 344 | resample = PILImageResampling.BICUBIC |
| 345 | size = {"height": 672, "width": 672} |
| 346 | default_to_square = False |
| 347 | do_rescale = True |
| 348 | rescale_factor = 1 / 255 |
| 349 | do_normalize = True |
| 350 | image_mean = [0.48145466, 0.4578275, 0.40821073] |
| 351 | image_std = [0.26862954, 0.26130258, 0.27577711] |
| 352 | do_convert_rgb = True |
| 353 | do_sample_frames = False |
| 354 | patch_size = 14 |
| 355 | temporal_patch_size = 2 |
| 356 | merge_size = 2 |
| 357 | min_pixels = 4 * 28 * 28 |
| 358 | max_pixels = 768 * 28 * 28 # 602,112 |
| 359 | total_pixels = int(64000 * 28 * 28 * 0.9) # ~45M, ~64k tokens budget |
| 360 | # Long-side resize spec (opt-in via ``max_long_side_pixel``). The video |
| 361 | # ``max_total_pixels`` cap is volumetric (width * height * frames) and is |
| 362 | # enforced in ``_preprocess`` once the frame count is known. |
| 363 | max_long_side_pixel = None |
| 364 | min_short_side_pixel = MIN_SHORT_SIDE_PIXEL |
| 365 | max_total_pixels = VIDEO_MAX_TOTAL_PIXELS |
| 366 | fps = 1.0 |
| 367 | min_frames = 4 |
| 368 | max_frames = 768 |
| 369 | valid_kwargs = MiniMaxM3VLVideoProcessorKwargs |
| 370 | model_input_names = ["pixel_values_videos", "video_grid_thw"] |
| 371 | |
| 372 | def __init__(self, **kwargs: Unpack[MiniMaxM3VLVideoProcessorKwargs]): |
| 373 | super().__init__(**kwargs) |
| 374 | |
| 375 | def _preprocess( |
| 376 | self, |
| 377 | videos: list[torch.Tensor], |
| 378 | do_resize: bool, |
| 379 | size: SizeDict, |
| 380 | resample: "PILImageResampling | InterpolationMode | int | None", |
| 381 | do_rescale: bool, |
| 382 | rescale_factor: float, |
| 383 | do_normalize: bool, |
| 384 | image_mean: "float | list[float] | None", |
| 385 | image_std: "float | list[float] | None", |
| 386 | patch_size: int, |
| 387 | temporal_patch_size: int, |
| 388 | merge_size: int, |
| 389 | min_pixels: int, |
| 390 | max_pixels: int, |
| 391 | max_long_side_pixel: "int | None" = None, |
| 392 | return_tensors: "str | TensorType | None" = None, |
| 393 | **kwargs, |
| 394 | ) -> BatchFeature: |
| 395 | grouped_videos, grouped_videos_index = group_videos_by_shape(videos) |
| 396 | resized_videos_grouped = {} |
| 397 | factor = patch_size * merge_size |
| 398 | for shape, stacked_videos in grouped_videos.items(): |
| 399 | batch_size, num_frames, channels, height, width = stacked_videos.shape |
| 400 | resized_height, resized_width = height, width |
| 401 | if do_resize: |
| 402 | resized_height, resized_width = smart_resize( |
| 403 | height, |
| 404 | width, |
| 405 | factor=factor, |
| 406 | min_pixels=min_pixels, |
| 407 | max_pixels=max_pixels, |
| 408 | max_long_side_pixel=max_long_side_pixel, |
| 409 | min_short_side_pixel=self.min_short_side_pixel, |
| 410 | # Per-frame raise disabled; the video cap is volumetric and |
| 411 | # is enforced below once num_frames is known. |
| 412 | max_total_pixels=None, |
| 413 | ) |
| 414 | if ( |
| 415 | max_long_side_pixel is not None |
| 416 | and resized_height * resized_width * num_frames |
| 417 | > self.max_total_pixels |
| 418 | ): |
| 419 | raise ValueError( |
| 420 | f"video area {resized_height * resized_width * num_frames} " |
| 421 | f"(width * height * frames) exceeds max_total_pixels " |
| 422 | f"{self.max_total_pixels} after resizing" |
| 423 | ) |
| 424 | stacked_videos = stacked_videos.view( |
| 425 | batch_size * num_frames, channels, height, width |
| 426 | ) |
| 427 | stacked_videos = self.resize( |
| 428 | stacked_videos, |
| 429 | size=SizeDict(height=resized_height, width=resized_width), |
| 430 | resample=resample, |
| 431 | ) |
| 432 | stacked_videos = stacked_videos.view( |
| 433 | batch_size, |
| 434 | num_frames, |
| 435 | channels, |
| 436 | resized_height, |
| 437 | resized_width, |
| 438 | ) |
| 439 | resized_videos_grouped[shape] = stacked_videos |
| 440 | resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index) |
| 441 | |
| 442 | grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos) |
| 443 | processed_videos_grouped = {} |
| 444 | processed_grids = {} |
| 445 | for shape, stacked_videos in grouped_videos.items(): |
| 446 | resized_height, resized_width = stacked_videos.shape[-2:] |
| 447 | patches = self.rescale_and_normalize( |
| 448 | stacked_videos, |
| 449 | do_rescale, |
| 450 | rescale_factor, |
| 451 | do_normalize, |
| 452 | image_mean, |
| 453 | image_std, |
| 454 | ) |
| 455 | |
| 456 | if pad := -patches.shape[1] % temporal_patch_size: |
| 457 | repeats = patches[:, -1:].expand(-1, pad, -1, -1, -1) |
| 458 | patches = torch.cat([patches, repeats], dim=1) |
| 459 | |
| 460 | batch_size, grid_t, channels = patches.shape[:3] |
| 461 | grid_t = grid_t // temporal_patch_size |
| 462 | grid_h, grid_w = resized_height // patch_size, resized_width // patch_size |
| 463 | |
| 464 | patches = patches.view( |
| 465 | batch_size, |
| 466 | grid_t, |
| 467 | temporal_patch_size, |
| 468 | channels, |
| 469 | grid_h // merge_size, |
| 470 | merge_size, |
| 471 | patch_size, |
| 472 | grid_w // merge_size, |
| 473 | merge_size, |
| 474 | patch_size, |
| 475 | ) |
| 476 | patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9) |
| 477 | flatten_patches = patches.reshape( |
| 478 | batch_size, |
| 479 | grid_t * grid_h * grid_w, |
| 480 | channels * temporal_patch_size * patch_size * patch_size, |
| 481 | ) |
| 482 | |
| 483 | processed_videos_grouped[shape] = flatten_patches |
| 484 | processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size |
| 485 | |
| 486 | processed_videos = reorder_videos( |
| 487 | processed_videos_grouped, grouped_videos_index |
| 488 | ) |
| 489 | processed_grids = reorder_videos(processed_grids, grouped_videos_index) |
| 490 | pixel_values_videos = torch.cat(processed_videos, dim=0) |
| 491 | video_grid_thw = torch.tensor(processed_grids, dtype=torch.long) |
| 492 | |
| 493 | return BatchFeature( |
| 494 | data={ |
| 495 | "pixel_values_videos": pixel_values_videos, |
| 496 | "video_grid_thw": video_grid_thw, |
| 497 | }, |
| 498 | tensor_type=return_tensors, |
| 499 | ) |
| 500 | |
| 501 | |
| 502 | class MiniMaxVLProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg] |
| 503 | _defaults = { |
| 504 | "videos_kwargs": { |
| 505 | "do_resize": False, |
| 506 | "return_metadata": True, |
| 507 | }, |
| 508 | } |
| 509 | |
| 510 | |
| 511 | class MiniMaxVLProcessor(ProcessorMixin): |
| 512 | IMAGE_TOKEN = "]<]image[>[" |
| 513 | VIDEO_TOKEN = "]<]video[>[" |
| 514 | VISION_START_TOKEN = "]<]start of image[>[" |
| 515 | VISION_END_TOKEN = "]<]end of image[>[" |
| 516 | |
| 517 | @classmethod |
| 518 | def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): |
| 519 | # Bypass ProcessorMixin's dynamic module lookup, which breaks in |
| 520 | # transformers >= 5.9 when image_processor_class is a string: the |
| 521 | # register() API now stores classes as {"pil": cls} dicts in |
| 522 | # _extra_content, but get_possibly_dynamic_module() still calls |
| 523 | # .__name__ on the raw value, crashing with AttributeError on dicts. |
| 524 | # |
| 525 | # Reuse the tokenizer passed by vLLM instead of loading another one, |
| 526 | # and keep it out of the kwargs forwarded to the sub-processor loaders. |
| 527 | tokenizer = kwargs.pop("tokenizer", None) |
| 528 | if tokenizer is None: |
| 529 | tokenizer = AutoTokenizer.from_pretrained( |
| 530 | pretrained_model_name_or_path, **kwargs |
| 531 | ) |
| 532 | image_processor = MiniMaxM3VLImageProcessor.from_pretrained( |
| 533 | pretrained_model_name_or_path, **kwargs |
| 534 | ) |
| 535 | video_processor = MiniMaxM3VLVideoProcessor.from_pretrained( |
| 536 | pretrained_model_name_or_path, **kwargs |
| 537 | ) |
| 538 | return cls( |
| 539 | image_processor=image_processor, |
| 540 | tokenizer=tokenizer, |
| 541 | video_processor=video_processor, |
| 542 | ) |
| 543 | |
| 544 | def __init__( |
| 545 | self, image_processor=None, tokenizer=None, video_processor=None, **kwargs |
| 546 | ): |
| 547 | self.image_token_id = tokenizer.convert_tokens_to_ids(self.IMAGE_TOKEN) |
| 548 | self.video_token_id = tokenizer.convert_tokens_to_ids(self.VIDEO_TOKEN) |
| 549 | super().__init__(image_processor, tokenizer, video_processor) |
| 550 | # Video expansion also uses image start/end tokens. Separate video |
| 551 | # start/end tokens exist in the tokenizer, but the original MiniMax |
| 552 | # serving path did not use them; keep that behavior for compatibility. |
| 553 | self.vision_start_token_id = tokenizer.convert_tokens_to_ids( |
| 554 | self.VISION_START_TOKEN |
| 555 | ) |
| 556 | self.vision_end_token_id = tokenizer.convert_tokens_to_ids( |
| 557 | self.VISION_END_TOKEN |
| 558 | ) |
| 559 | |
| 560 | def __call__( |
| 561 | self, |
| 562 | images=None, |
| 563 | text=None, |
| 564 | videos=None, |
| 565 | **kwargs: Unpack[MiniMaxVLProcessorKwargs], |
| 566 | ) -> BatchFeature: |
| 567 | output_kwargs = self._merge_kwargs( |
| 568 | MiniMaxVLProcessorKwargs, |
| 569 | tokenizer_init_kwargs=self.tokenizer.init_kwargs, |
| 570 | **kwargs, |
| 571 | ) |
| 572 | |
| 573 | if images is not None: |
| 574 | images_kwargs = output_kwargs["images_kwargs"] |
| 575 | image_inputs = self.image_processor(images=images, **images_kwargs) |
| 576 | image_grid_thw = image_inputs["image_grid_thw"] |
| 577 | else: |
| 578 | image_inputs = {} |
| 579 | image_grid_thw = None |
| 580 | |
| 581 | if videos is not None: |
| 582 | videos_kwargs = output_kwargs["videos_kwargs"] |
| 583 | video_inputs = self.video_processor(videos=videos, **videos_kwargs) |
| 584 | video_grid_thw = video_inputs["video_grid_thw"] |
| 585 | if not kwargs.get("return_metadata"): |
| 586 | video_metadata = video_inputs.pop("video_metadata") |
| 587 | else: |
| 588 | video_metadata = video_inputs["video_metadata"] |
| 589 | else: |
| 590 | video_inputs = {} |
| 591 | video_grid_thw = None |
| 592 | |
| 593 | if not isinstance(text, list): |
| 594 | text = [text] |
| 595 | text = text.copy() |
| 596 | |
| 597 | # Expand image tokens |
| 598 | if image_grid_thw is not None: |
| 599 | merge_length = self.image_processor.merge_size**2 |
| 600 | placeholder = "]<]placeholder[>[" |
| 601 | index = 0 |
| 602 | for i in range(len(text)): |
| 603 | while self.IMAGE_TOKEN in text[i]: |
| 604 | num_tokens = image_grid_thw[index].prod() // merge_length |
| 605 | text[i] = text[i].replace( |
| 606 | self.IMAGE_TOKEN, |
| 607 | self.VISION_START_TOKEN |
| 608 | + placeholder * num_tokens |
| 609 | + self.VISION_END_TOKEN, |
| 610 | 1, |
| 611 | ) |
| 612 | index += 1 |
| 613 | text[i] = text[i].replace(placeholder, self.IMAGE_TOKEN) |
| 614 | |
| 615 | # Expand video tokens |
| 616 | if video_grid_thw is not None: |
| 617 | merge_length = self.image_processor.merge_size**2 |
| 618 | placeholder = "]<]placeholder[>[" |
| 619 | index = 0 |
| 620 | for i in range(len(text)): |
| 621 | while self.VIDEO_TOKEN in text[i]: |
| 622 | metadata = video_metadata[index] |
| 623 | grid_t = video_grid_thw[index][0] |
| 624 | frame_seqlen = video_grid_thw[index][1:].prod() // merge_length |
| 625 | |
| 626 | video_placeholder = "" |
| 627 | for frame_idx in range(grid_t): |
| 628 | if ( |
| 629 | metadata.fps is not None |
| 630 | and metadata.frames_indices is not None |
| 631 | ): |
| 632 | ts = ( |
| 633 | metadata.frames_indices[ |
| 634 | min( |
| 635 | frame_idx |
| 636 | * self.video_processor.temporal_patch_size, |
| 637 | len(metadata.frames_indices) - 1, |
| 638 | ) |
| 639 | ] |
| 640 | / metadata.fps |
| 641 | ) |
| 642 | video_placeholder += f"]<]{ts:.1f} seconds[>[" |
| 643 | video_placeholder += ( |
| 644 | self.VISION_START_TOKEN |
| 645 | + placeholder * frame_seqlen |
| 646 | + self.VISION_END_TOKEN |
| 647 | ) |
| 648 | |
| 649 | text[i] = text[i].replace(self.VIDEO_TOKEN, video_placeholder, 1) |
| 650 | index += 1 |
| 651 | text[i] = text[i].replace(placeholder, self.VIDEO_TOKEN) |
| 652 | |
| 653 | # Tokenize |
| 654 | return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None) |
| 655 | text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"]) |
| 656 | |
| 657 | return BatchFeature( |
| 658 | data={**text_inputs, **image_inputs, **video_inputs}, |
| 659 | tensor_type=return_tensors, |
| 660 | ) |