@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cde384cd92
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 12:15 UTC

[Model] support MiniMax-VL-01 model (#16328)

Signed-off-by: qingjun <qingjun@minimaxi.com>

qscqesze committed Apr 29, 2025 at 12:05 UTC cde384cd92c811c2237cf21681166fd41437c8a3
11 files changed +954 -19
tests/models/decoder_only/vision_language/test_models.py
+13
@@ -446,6 +446,19 @@ VLM_TEST_SETTINGS = {
446 hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
447 patch_hf_runner=model_utils.minicpmv_26_patch_hf_runner,
448 ),
449 + "minimax_vl_01": VLMTestInfo(
450 + models=["MiniMaxAI/MiniMax-VL-01"],
451 + prompt_formatter=lambda img_prompt: f"<beginning_of_sentence>user: {img_prompt} assistant:<end_of_sentence>", # noqa: E501
452 + img_idx_to_prompt=lambda _: "<image>",
453 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
454 + max_model_len=8192,
455 + max_num_seqs=4,
456 + dtype="bfloat16",
457 + hf_output_post_proc=model_utils.minimax_vl_01_hf_output,
458 + patch_hf_runner=model_utils.minimax_vl_01_patch_hf_runner,
459 + auto_cls=AutoModelForImageTextToText,
460 + marks=[large_gpu_mark(min_gb=80)],
461 + ),
462 "molmo": VLMTestInfo(
463 models=["allenai/Molmo-7B-D-0924"],
464 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+19
@@ -229,6 +229,14 @@ def minicpmv_trunc_hf_output(hf_output: RunnerOutput,
229 return output_ids, output_str, out_logprobs
230
231
232 +def minimax_vl_01_hf_output(hf_output: RunnerOutput,
233 + model: str) -> RunnerOutput:
234 + output_ids, output_str, out_logprobs = hf_output
235 + if output_str.endswith("<end_of_sentence>"):
236 + output_str = output_str.split("<end_of_sentence>")[0]
237 + return output_ids, output_str, out_logprobs
238 +
239 +
240 ####### Functions for converting image assets to embeddings
241 def get_llava_embeddings(image_assets: _ImageAssets):
242 return [asset.image_embeds for asset in image_assets]
@@ -627,6 +635,17 @@ def minicpmv_26_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
635 return hf_model
636
637
638 +def minimax_vl_01_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
639 + orig_generate = hf_model.model.generate
640 +
641 + def _generate(self, *args, image_sizes=None, **kwargs):
642 + return orig_generate(*args, decode_text=False, **kwargs)
643 +
644 + hf_model.model.generate = types.MethodType(_generate, hf_model.model)
645 +
646 + return hf_model
647 +
648 +
649 def molmo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
650 """Patches and returns an instance of the HfRunner to use for Molmo."""
651 hf_processor = hf_model.processor
tests/models/multimodal/processing/test_minimax_vl_01.py new
+99
@@ -0,0 +1,99 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +import pytest
4 +from PIL import Image
5 +
6 +from vllm.multimodal import MULTIMODAL_REGISTRY
7 +from vllm.multimodal.parse import ImageSize
8 +from vllm.multimodal.processing import BaseMultiModalProcessor
9 +
10 +from ....conftest import _ImageAssets
11 +from ...utils import build_model_context
12 +
13 +
14 +@pytest.mark.parametrize("model_id", ["MiniMaxAI/MiniMax-VL-01"])
15 +# yapf: enable
16 +@pytest.mark.parametrize("num_imgs", [1, 2])
17 +def test_processor_override(
18 + image_assets: _ImageAssets,
19 + model_id: str,
20 + num_imgs: int,
21 +):
22 + ctx = build_model_context(
23 + model_id,
24 + mm_processor_kwargs=None,
25 + limit_mm_per_prompt={"image": num_imgs},
26 + )
27 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
28 + prompt = "<image>" * num_imgs
29 + image = Image.new("RGB", size=(364, 364))
30 + mm_data = {"image": [image] * num_imgs}
31 +
32 + processed_inputs = processor.apply(prompt, mm_data, {})
33 + image_placeholders = processed_inputs["mm_placeholders"]["image"]
34 +
35 + assert len(image_placeholders) == num_imgs
36 +
37 +
38 +def _validate_image_prompt_replacements_one(
39 + processor: BaseMultiModalProcessor,
40 + num_imgs: int,
41 + failed_size_excs: list[tuple[ImageSize, Exception]],
42 + image_size: ImageSize,
43 +) -> None:
44 + prompt = "<image>" * num_imgs
45 + image = Image.new("RGB", size=image_size)
46 + mm_data = {"image": [image] * num_imgs}
47 +
48 + try:
49 + processed_inputs = processor.apply(prompt, mm_data, {})
50 +
51 + image_placeholders = processed_inputs["mm_placeholders"]["image"]
52 + assert len(image_placeholders) == num_imgs
53 +
54 + except Exception as exc:
55 + failed_size_excs.append((image_size, exc))
56 +
57 +
58 +def _test_image_prompt_replacements(
59 + processor,
60 + *,
61 + num_imgs: int,
62 + image_sizes: list[ImageSize],
63 +) -> None:
64 +
65 + failed_size_excs = list[tuple[ImageSize, Exception]]()
66 +
67 + for size in image_sizes:
68 + _validate_image_prompt_replacements_one(processor, num_imgs,
69 + failed_size_excs, size)
70 +
71 + if failed_size_excs:
72 + msg = "Found failing image sizes:" \
73 + + "\n========\n".join(f"[{size}]\n{exc}"
74 + for size, exc in failed_size_excs)
75 + raise AssertionError(msg)
76 +
77 +
78 +@pytest.mark.parametrize("model_id", ["MiniMaxAI/MiniMax-VL-01"])
79 +@pytest.mark.parametrize("num_imgs", [1, 2])
80 +def test_processor_prompt_replacements_regression(model_id, num_imgs):
81 + ctx = build_model_context(
82 + model_id,
83 + mm_processor_kwargs=None,
84 + limit_mm_per_prompt={"image": num_imgs},
85 + )
86 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
87 +
88 + image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),
89 + (488, 183), (2560, 1669)]
90 + image_sizes = [
91 + size for w, h in image_ratios
92 + for size in [ImageSize(w, h), ImageSize(h, w)]
93 + ]
94 +
95 + _test_image_prompt_replacements(
96 + processor,
97 + num_imgs=num_imgs,
98 + image_sizes=image_sizes,
99 + )
tests/models/registry.py
+2
@@ -337,6 +337,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
337 "MiniCPMV": _HfExamplesInfo("openbmb/MiniCPM-Llama3-V-2_5",
338 extras={"2.6": "openbmb/MiniCPM-V-2_6"}, # noqa: E501
339 trust_remote_code=True),
340 + "MiniMaxVL01ForConditionalGeneration": _HfExamplesInfo("MiniMaxAI/MiniMax-VL-01", # noqa: E501
341 + trust_remote_code=True),
342 "Mistral3ForConditionalGeneration": _HfExamplesInfo("mistralai/Mistral-Small-3.1-24B-Instruct-2503", # noqa: E501
343 extras={"fp8": "nm-testing/Mistral-Small-3.1-24B-Instruct-2503-FP8-dynamic"}), # noqa: E501
344 "MolmoForCausalLM": _HfExamplesInfo("allenai/Molmo-7B-D-0924",
vllm/model_executor/models/minimax_text_01.py
+53 -14
@@ -3,7 +3,7 @@
3 import copy
4 import math
5 import re
6 -from typing import Dict, Iterable, List, Optional, Tuple, Union
6 +from typing import Dict, Iterable, List, Optional, Set, Tuple, Union
7
8 import torch
9 import torch.distributed
@@ -110,7 +110,17 @@ class MiniMaxText01RMSNormTP(CustomOp):
110 variance = tensor_model_parallel_all_reduce(
111 variance) / self.tp_world
112 x = x * torch.rsqrt(variance + self.variance_epsilon)
113 - x = x.to(orig_dtype) * self.weight
113 +
114 + weight = self.weight
115 + if x.size(-1) != self.weight.size(0):
116 + if self.weight.size(0) < x.size(-1):
117 + repeat_count = (x.size(-1) + self.weight.size(0)) // x.size(-1)
118 + full_weight = self.weight.repeat(repeat_count)
119 + weight = full_weight[:x.size(-1)]
120 + else:
121 + weight = self.weight[:x.size(-1)]
122 +
123 + x = x.to(orig_dtype) * weight
124 return x
125
126 def forward(
@@ -421,6 +431,10 @@ class MiniMaxText01LinearAttention(nn.Module):
431 attn_metadata):
432 hidden = []
433 for _prefill_idx in range(getattr(attn_metadata, "num_prefills", 0)):
434 + if _prefill_idx >= len(attn_metadata.query_start_loc):
435 + break
436 + if _prefill_idx >= len(state_indices_tensor):
437 + break
438 _start = attn_metadata.query_start_loc[_prefill_idx]
439 _end = attn_metadata.query_start_loc[_prefill_idx + 1]
440 slot_id = state_indices_tensor[_prefill_idx]
@@ -443,6 +457,10 @@ class MiniMaxText01LinearAttention(nn.Module):
457 hidden.append(
458 self._decode_infer(q, k, v, kv_cache, state_indices_tensor,
459 attn_metadata))
460 +
461 + if not hidden:
462 + return torch.empty((0, q.size(-1)), device=q.device, dtype=q.dtype)
463 +
464 hidden = torch.concat(hidden, dim=0).contiguous()
465 return hidden
466
@@ -663,6 +681,9 @@ class MiniMaxText01DecoderLayer(nn.Module):
681 self.shared_moe = False
682
683 shared_intermediate = getattr(config, 'shared_intermediate_size', 0)
684 + if isinstance(shared_intermediate, list):
685 + shared_intermediate = shared_intermediate[
686 + layer_id] if layer_id < len(shared_intermediate) else 0
687 if shared_intermediate > 0:
688 self.shared_moe = True
689 self.shared_mlp = MiniMaxText01MLP(
@@ -875,6 +896,8 @@ class MiniMaxText01Model(nn.Module):
896
897 slots_to_clear = []
898 for _prefill_id in range(getattr(attn_metadata, "num_prefills", 0)):
899 + if _prefill_id >= len(seq_id_map):
900 + break
901 seq_id = seq_id_map[_prefill_id]
902 if attn_metadata.context_lens_tensor[
903 _prefill_id] == 0 and seq_id in seq_to_slot_maps:
@@ -886,13 +909,18 @@ class MiniMaxText01Model(nn.Module):
909 dtype=torch.long)
910 minimax_cache_tensors[:, slots_tensor, ...] = 0
911
912 + def get_input_embeddings(
913 + self,
914 + input_ids: torch.Tensor,
915 + ) -> torch.Tensor:
916 + return self.embed_tokens(input_ids)
917 +
918 def forward(self,
919 input_ids: Optional[torch.Tensor],
920 positions: torch.Tensor,
892 - kv_caches: List[torch.Tensor],
893 - intermediate_tensors=None,
921 + intermediate_tensors: Optional[IntermediateTensors] = None,
922 inputs_embeds: Optional[torch.Tensor] = None,
895 - **kwargs) -> torch.Tensor:
923 + **kwargs) -> Union[torch.Tensor, IntermediateTensors]:
924 forward_context = get_forward_context()
925 attn_metadata = forward_context.attn_metadata
926 if attn_metadata is None:
@@ -901,6 +929,7 @@ class MiniMaxText01Model(nn.Module):
929 kwargs["request_ids_to_seq_ids"] = {}
930 if "finished_requests_ids" not in kwargs:
931 kwargs["finished_requests_ids"] = []
932 +
933 (
934 minimax_cache_tensors,
935 state_indices_tensor,
@@ -922,15 +951,11 @@ class MiniMaxText01Model(nn.Module):
951 hidden_states = intermediate_tensors["hidden_states"]
952 residual = intermediate_tensors["residual"]
953
925 - kv_cache_index = 0
954 minimax_cache_index = 0
955 attn_metadata.rotary_emb = self.rotary_emb
956 for i in range(self.start_layer, self.end_layer):
957 layer = self.layers[i]
958 _caches = None
931 - if isinstance(layer.self_attn, MiniMaxText01Attention):
932 - _caches = kv_caches[kv_cache_index]
933 - kv_cache_index += 1
959 if isinstance(layer.self_attn, MiniMaxText01LinearAttention):
960 current_state_layer = minimax_cache_index
961 _caches = minimax_cache_params.at_layer_idx(
@@ -1009,15 +1034,20 @@ class MiniMaxText01ForCausalLM(nn.Module, HasInnerState, IsHybrid,
1034 return self.model.minimax_cache.get_seqlen_agnostic_capture_inputs(
1035 batch_size)
1036
1037 + def get_input_embeddings(
1038 + self,
1039 + input_ids: torch.Tensor,
1040 + ) -> torch.Tensor:
1041 + return self.model.get_input_embeddings(input_ids)
1042 +
1043 def forward(self,
1044 input_ids: torch.Tensor,
1045 positions: torch.Tensor,
1046 intermediate_tensors: Optional[IntermediateTensors] = None,
1047 inputs_embeds: Optional[torch.Tensor] = None,
1048 **kwargs) -> torch.Tensor:
1018 - hidden_states = self.model(input_ids, positions, self.kv_cache,
1019 - intermediate_tensors, inputs_embeds,
1020 - **kwargs)
1049 + hidden_states = self.model(input_ids, positions, intermediate_tensors,
1050 + inputs_embeds, **kwargs)
1051
1052 return hidden_states
1053
@@ -1043,8 +1073,9 @@ class MiniMaxText01ForCausalLM(nn.Module, HasInnerState, IsHybrid,
1073 })
1074
1075 def load_weights(self, weights: Iterable[Tuple[str,
1046 - torch.Tensor]]) -> None:
1076 + torch.Tensor]]) -> Set[str]:
1077 params_dict = dict(self.named_parameters())
1078 + loaded_params: Set[str] = set()
1079
1080 def which_layer(name: str) -> int:
1081 if "layers" in name:
@@ -1108,6 +1139,7 @@ class MiniMaxText01ForCausalLM(nn.Module, HasInnerState, IsHybrid,
1139 weight_name,
1140 expert_id=expert_id,
1141 shard_id=shard_id)
1142 + loaded_params.add(name)
1143 break
1144 else:
1145 if is_pp_missing_parameter(name, self):
@@ -1117,6 +1149,7 @@ class MiniMaxText01ForCausalLM(nn.Module, HasInnerState, IsHybrid,
1149 default_weight_loader)
1150 weight_loader = weight_loader_with_alias(name)(weight_loader)
1151 weight_loader(param, loaded_weight)
1152 + loaded_params.add(name)
1153 return
1154
1155 def is_shared_mlp_weight(name: str) -> bool:
@@ -1154,6 +1187,7 @@ class MiniMaxText01ForCausalLM(nn.Module, HasInnerState, IsHybrid,
1187 else:
1188 raise AssertionError(
1189 "MLP weight not in [gate_up_proj, down_proj]")
1190 + loaded_params.add(name)
1191 return
1192
1193 def is_mha_weight(name: str) -> bool:
@@ -1170,6 +1204,7 @@ class MiniMaxText01ForCausalLM(nn.Module, HasInnerState, IsHybrid,
1204 MiniMaxText01LinearAttention.weight_direct_load)
1205 weight_loader = weight_loader_with_alias(name)(weight_loader)
1206 weight_loader(param, loaded_weight)
1207 + loaded_params.add(name)
1208 return
1209
1210 def load_flash_attn_weight(name: str, loaded_weight: torch.Tensor,
@@ -1194,6 +1229,7 @@ class MiniMaxText01ForCausalLM(nn.Module, HasInnerState, IsHybrid,
1229 default_weight_loader)
1230 weight_loader = weight_loader_with_alias(name)(weight_loader)
1231 weight_loader(param, loaded_weight, shard_id)
1232 + loaded_params.add(name)
1233 break
1234 else:
1235 if is_pp_missing_parameter(name, self):
@@ -1204,6 +1240,7 @@ class MiniMaxText01ForCausalLM(nn.Module, HasInnerState, IsHybrid,
1240 default_weight_loader)
1241 weight_loader = weight_loader_with_alias(name)(weight_loader)
1242 weight_loader(param, loaded_weight)
1243 + loaded_params.add(name)
1244 return
1245
1246 def is_layer_norm_weight(name: str) -> bool:
@@ -1219,6 +1256,7 @@ class MiniMaxText01ForCausalLM(nn.Module, HasInnerState, IsHybrid,
1256 default_weight_loader)
1257 weight_loader = weight_loader_with_alias(name)(weight_loader)
1258 weight_loader(param, loaded_weight)
1259 + loaded_params.add(name)
1260 return
1261
1262 def load_basic_weight(name: str, loaded_weight: torch.Tensor,
@@ -1230,6 +1268,7 @@ class MiniMaxText01ForCausalLM(nn.Module, HasInnerState, IsHybrid,
1268 default_weight_loader)
1269 weight_loader = weight_loader_with_alias(name)(weight_loader)
1270 weight_loader(param, loaded_weight)
1271 + loaded_params.add(name)
1272 return
1273
1274 for name, loaded_weight in weights:
@@ -1258,4 +1297,4 @@ class MiniMaxText01ForCausalLM(nn.Module, HasInnerState, IsHybrid,
1297 continue
1298
1299 load_basic_weight(name, loaded_weight, self)
1261 - return
1300 + return loaded_params
vllm/model_executor/models/minimax_vl_01.py new
+615
@@ -0,0 +1,615 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +from abc import abstractmethod
4 +from collections.abc import Iterable, Mapping, Sequence
5 +from dataclasses import dataclass
6 +from typing import (Final, Literal, Optional, Protocol, Set, Tuple, TypedDict,
7 + TypeVar, Union, cast)
8 +
9 +import numpy as np
10 +import torch
11 +import torch.nn as nn
12 +from transformers import BatchFeature, CLIPVisionConfig, PretrainedConfig
13 +from transformers.image_processing_utils import select_best_resolution
14 +
15 +from vllm.config import VllmConfig
16 +from vllm.jsontree import json_map_leaves
17 +from vllm.logger import init_logger
18 +from vllm.model_executor.layers.activation import get_act_fn
19 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
20 + RowParallelLinear)
21 +from vllm.model_executor.layers.quantization import QuantizationConfig
22 +from vllm.model_executor.sampling_metadata import SamplingMetadata
23 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
24 +from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
25 +from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
26 + ImageSize, MultiModalDataItems)
27 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
28 + BaseProcessingInfo, PromptReplacement,
29 + PromptUpdate)
30 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
31 +from vllm.sequence import IntermediateTensors
32 +from vllm.transformers_utils.configs.minimax_vl_01 import MiniMaxVL01Config
33 +
34 +from .clip import CLIPVisionModel
35 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
36 +from .pixtral import PixtralHFVisionModel
37 +from .siglip import SiglipVisionModel
38 +from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
39 + maybe_prefix, merge_multimodal_embeddings)
40 +from .vision import get_vision_encoder_info
41 +
42 +logger = init_logger(__name__)
43 +
44 +
45 +# For dummy input only
46 +@dataclass
47 +class MaxImageTokenMeta:
48 + width: int = 1024
49 + height: int = 1024
50 +
51 +
52 +class MiniMaxVL01ImagePixelInputs(TypedDict):
53 + type: Literal["pixel_values"]
54 + pixel_values: torch.Tensor
55 + """
56 + Shape: `(batch_size * num_images, num_channels, height, width)`
57 +
58 + Note that `height` or `width` may be different per batch and image,
59 + in which case the data is passed as a list instead of a batched tensor.
60 + """
61 +
62 +
63 +class MiniMaxVL01ImageEmbeddingInputs(TypedDict):
64 + type: Literal["image_embeds"]
65 + data: torch.Tensor
66 + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
67 +
68 + `hidden_size` must match the hidden size of language model backbone.
69 + """
70 +
71 +
72 +def image_size_to_num_patches(image_size, grid_pinpoints, patch_size: int):
73 + if not isinstance(grid_pinpoints, list):
74 + raise TypeError("grid_pinpoints should be a list of tuples or lists")
75 +
76 + # ! VERY IMPORTANT if image_size is tensor, must convert to into tuple,
77 + # otherwise it will cause wrong calculate
78 + if not isinstance(image_size, (list, tuple)):
79 + if not isinstance(image_size, (torch.Tensor, np.ndarray)):
80 + raise TypeError("image_size invalid type " +
81 + f"{type(image_size)} with value {image_size}")
82 + image_size = image_size.tolist()
83 +
84 + best_resolution = select_best_resolution(image_size, grid_pinpoints)
85 + height, width = best_resolution
86 + num_patches = 0
87 + # consider change to ceil(height/patch_size)*ceil(width/patch_size) + 1
88 + for i in range(0, height, patch_size):
89 + for j in range(0, width, patch_size):
90 + num_patches += 1
91 + # add the base patch
92 + num_patches += 1
93 + return num_patches
94 +
95 +
96 +def get_anyres_image_grid_shape(image_size, grid_pinpoints, patch_size):
97 + if not isinstance(grid_pinpoints, list):
98 + raise TypeError("grid_pinpoints should be a list of tuples or lists")
99 +
100 + # ! VERY IMPORTANT if image_size is tensor,
101 + # must convert to into tuple,
102 + # otherwise it will cause wrong calculate
103 + if not isinstance(image_size, (list, tuple)):
104 + if not isinstance(image_size, (torch.Tensor, np.ndarray)):
105 + raise TypeError(
106 + "image_size invalid type " +
107 + f"{type(image_size)} not valid, " +
108 + "should be either list, tuple, np.ndarray or tensor")
109 + image_size = image_size.tolist()
110 +
111 + height, width = select_best_resolution(image_size, grid_pinpoints)
112 + return height // patch_size, width // patch_size
113 +
114 +
115 +def unpad_image(tensor, original_size):
116 + original_height, original_width = original_size
117 + current_height, current_width = tensor.shape[1:]
118 +
119 + original_aspect_ratio = original_width / original_height
120 + current_aspect_ratio = current_width / current_height
121 +
122 + if original_aspect_ratio > current_aspect_ratio:
123 + new_height = int(original_height * current_width) // original_width
124 + padding = (current_height - new_height) // 2
125 + unpadded_tensor = tensor[:, padding:current_height - padding, :]
126 + else:
127 + new_width = int(original_width * current_height) // original_height
128 + padding = (current_width - new_width) // 2
129 + unpadded_tensor = tensor[:, :, padding:current_width - padding]
130 +
131 + return unpadded_tensor
132 +
133 +
134 +class MiniMaxVL01MultiModalProjector(nn.Module):
135 +
136 + def __init__(self,
137 + vision_hidden_size: int,
138 + text_hidden_size: int,
139 + projector_hidden_act: str,
140 + multimodal_projector_bias: bool,
141 + quant_config: Optional[QuantizationConfig] = None,
142 + prefix: str = ""):
143 + super().__init__()
144 +
145 + self.linear_1 = ColumnParallelLinear(vision_hidden_size,
146 + text_hidden_size,
147 + bias=multimodal_projector_bias,
148 + quant_config=quant_config,
149 + prefix=f"{prefix}.linear_1")
150 + self.act = get_act_fn(projector_hidden_act)
151 + self.linear_2 = RowParallelLinear(text_hidden_size,
152 + text_hidden_size,
153 + bias=multimodal_projector_bias,
154 + quant_config=quant_config,
155 + prefix=f"{prefix}.linear_2")
156 +
157 + def forward(self, image_features: torch.Tensor) -> torch.Tensor:
158 + hidden_states, _ = self.linear_1(image_features)
159 + hidden_states = self.act(hidden_states)
160 + hidden_states, _ = self.linear_2(hidden_states)
161 + return hidden_states
162 +
163 +
164 +class MiniMaxVL01LikeConfig(Protocol):
165 + vision_config: Final[PretrainedConfig]
166 + image_token_index: Final[int]
167 + vision_feature_select_strategy: Final[str]
168 + vision_feature_layer: Final[Union[int, list[int]]]
169 +
170 +
171 +class MiniMaxVL01LikeProcessor(Protocol):
172 + image_token: Final[str]
173 +
174 +
175 +_I = TypeVar("_I", bound=BaseProcessingInfo)
176 +
177 +
178 +class MiniMaxVL01DummyInputsBuilder(BaseDummyInputsBuilder[_I]):
179 +
180 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
181 + num_images = mm_counts.get("image", 0)
182 + processor = self.info.get_hf_processor()
183 + image_token = processor.image_token
184 + return image_token * num_images
185 +
186 + def get_dummy_mm_data(
187 + self,
188 + seq_len: int,
189 + mm_counts: Mapping[str, int],
190 + ) -> MultiModalDataDict:
191 + num_images = mm_counts.get("image", 0)
192 +
193 + return {
194 + "image":
195 + self._get_dummy_images(width=MaxImageTokenMeta.width,
196 + height=MaxImageTokenMeta.height,
197 + num_images=num_images)
198 + }
199 +
200 +
201 +class MiniMaxVL01ProcessingInfo(BaseProcessingInfo):
202 +
203 + def get_hf_config(self):
204 + return self.ctx.get_hf_config(MiniMaxVL01Config)
205 +
206 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
207 + return {"image": None}
208 +
209 + def get_vision_encoder_info(self):
210 + return get_vision_encoder_info(self.get_hf_config())
211 +
212 + def _apply_feature_select_strategy(
213 + self,
214 + strategy: str,
215 + encoder_num_image_tokens: int,
216 + ) -> int:
217 + if strategy == "default":
218 + return encoder_num_image_tokens - 1
219 + if strategy == "full":
220 + return encoder_num_image_tokens
221 +
222 + msg = f"Unexpected feature select strategy: {strategy!r}"
223 + raise NotImplementedError(msg)
224 +
225 + def get_num_image_tokens(
226 + self,
227 + *,
228 + image_width: int,
229 + image_height: int,
230 + ) -> int:
231 + hf_config = self.get_hf_config()
232 + vision_encoder_info = self.get_vision_encoder_info()
233 +
234 + return self._apply_feature_select_strategy(
235 + hf_config.vision_feature_select_strategy,
236 + vision_encoder_info.get_num_image_tokens(
237 + image_width=image_width,
238 + image_height=image_height,
239 + ),
240 + )
241 +
242 + def get_image_size_with_most_features(self) -> ImageSize:
243 + vision_encoder_info = self.get_vision_encoder_info()
244 + width = height = vision_encoder_info.get_image_size()
245 + return ImageSize(width=width, height=height)
246 +
247 + def get_max_image_tokens(self) -> int:
248 + target_width, target_height = self.get_image_size_with_most_features()
249 +
250 + return self.get_num_image_tokens(
251 + image_width=target_width,
252 + image_height=target_height,
253 + )
254 +
255 +
256 +class BaseMiniMaxVL01MultiModalProcessor(BaseMultiModalProcessor[_I]):
257 +
258 + # Copied from BaseMultiModalProcessor
259 + @abstractmethod
260 + def _get_mm_fields_config(
261 + self,
262 + hf_inputs: BatchFeature,
263 + hf_processor_mm_kwargs: Mapping[str, object],
264 + ) -> Mapping[str, MultiModalFieldConfig]:
265 + raise NotImplementedError
266 +
267 + def _get_prompt_updates(
268 + self,
269 + mm_items: MultiModalDataItems,
270 + hf_processor_mm_kwargs: Mapping[str, object],
271 + out_mm_kwargs: MultiModalKwargs,
272 + ) -> Sequence[PromptUpdate]:
273 + hf_config = self.info.get_hf_config()
274 + image_token_id = hf_config.image_token_index
275 +
276 + def get_replacement(item_idx: int):
277 + images = mm_items.get_items(
278 + "image", (ImageEmbeddingItems, ImageProcessorItems))
279 +
280 + if isinstance(images, ImageEmbeddingItems):
281 + num_image_tokens = images.get_feature_size(item_idx)
282 + else:
283 + image_size = images.get_image_size(item_idx)
284 + num_image_tokens = self.info.get_num_image_tokens(
285 + image_width=image_size.width,
286 + image_height=image_size.height,
287 + )
288 +
289 + return [image_token_id] * num_image_tokens
290 +
291 + return [
292 + PromptReplacement(
293 + modality="image",
294 + target=[image_token_id],
295 + replacement=get_replacement,
296 + ),
297 + ]
298 +
299 +
300 +class MiniMaxVL01MultiModalProcessor(
301 + BaseMiniMaxVL01MultiModalProcessor[MiniMaxVL01ProcessingInfo]):
302 +
303 + def _call_hf_processor(
304 + self,
305 + prompt: str,
306 + mm_data: Mapping[str, object],
307 + mm_kwargs: Mapping[str, object],
308 + ) -> BatchFeature:
309 + processed_outputs = super()._call_hf_processor(
310 + prompt=prompt,
311 + mm_data=mm_data,
312 + mm_kwargs=mm_kwargs,
313 + )
314 +
315 + pixel_values = processed_outputs.get("pixel_values")
316 + if pixel_values is not None:
317 + image_sizes = processed_outputs["image_sizes"]
318 + min_len = min(len(pixel_values), len(image_sizes))
319 + pixel_values = pixel_values[:min_len]
320 + image_sizes = image_sizes[:min_len]
321 + assert len(pixel_values) == len(image_sizes)
322 +
323 + processed_outputs["pixel_values"] = [
324 + p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
325 + ]
326 +
327 + return processed_outputs
328 +
329 + def _get_mm_fields_config(
330 + self,
331 + hf_inputs: BatchFeature,
332 + hf_processor_mm_kwargs: Mapping[str, object],
333 + ) -> Mapping[str, MultiModalFieldConfig]:
334 + return {
335 + "pixel_values": MultiModalFieldConfig.batched("image"),
336 + "image_embeds": MultiModalFieldConfig.batched("image"),
337 + }
338 +
339 +
340 +def _get_num_hidden_layers(hf_config: MiniMaxVL01LikeConfig) -> int:
341 + """Determine the number of hidden layers to initialize up to in the
342 + visual encoder.
343 +
344 + Args:
345 + hf_config: Model config with vision feature layer(s).
346 + """
347 + feature_layers = hf_config.vision_feature_layer
348 + num_hidden_layers = hf_config.vision_config.num_hidden_layers
349 + # If we have one feature layer, initialize up to that layer
350 + if isinstance(feature_layers, int):
351 + return _get_layer_index(feature_layers, num_hidden_layers)
352 + # If we have multiple feature layers, initialize up to the deepest one
353 + elif isinstance(feature_layers, (list, tuple)):
354 + return max(
355 + _get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
356 + raise TypeError(f"vision_layer_feature type: {type(feature_layers)}"
357 + " is not supported")
358 +
359 +
360 +def _get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int:
361 + """Given a signed vision feature layer, get the number of hidden layers
362 + needed to leverage it.
363 +
364 + Args:
365 + feature_layer_index: Index of a required layer in the visual encoder.
366 + num_hidden_layers: The total number of hidden layers in the visual
367 + encoder.
368 + """
369 + if feature_layer_index < 0:
370 + return num_hidden_layers + feature_layer_index + 1
371 + return feature_layer_index
372 +
373 +
374 +def init_vision_tower_for_MiniMaxVL01(
375 + hf_config: MiniMaxVL01LikeConfig,
376 + quant_config: Optional[QuantizationConfig],
377 + *,
378 + require_post_norm: Optional[bool] = None,
379 + prefix: str = "",
380 +) -> Union[CLIPVisionModel, SiglipVisionModel, PixtralHFVisionModel]:
381 + vision_config = hf_config.vision_config
382 +
383 + # Initialize the vision tower only up to the deepest required feature layer
384 + num_hidden_layers = _get_num_hidden_layers(hf_config)
385 +
386 + if isinstance(vision_config, CLIPVisionConfig):
387 + return CLIPVisionModel(
388 + vision_config,
389 + quant_config=quant_config,
390 + num_hidden_layers_override=num_hidden_layers,
391 + require_post_norm=require_post_norm,
392 + prefix=prefix,
393 + )
394 +
395 + msg = f"Unsupported vision config: {type(vision_config)}"
396 + raise NotImplementedError(msg)
397 +
398 +
399 +@MULTIMODAL_REGISTRY.register_processor(
400 + MiniMaxVL01MultiModalProcessor,
401 + info=MiniMaxVL01ProcessingInfo,
402 + dummy_inputs=MiniMaxVL01DummyInputsBuilder)
403 +class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal,
404 + SupportsPP):
405 +
406 + packed_modules_mapping = {
407 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],
408 + "gate_up_proj": ["gate_proj", "up_proj"]
409 + }
410 +
411 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
412 + super().__init__()
413 +
414 + config = vllm_config.model_config.hf_config
415 + quant_config = vllm_config.quant_config
416 + multimodal_config = vllm_config.model_config.multimodal_config
417 +
418 + self.config = config
419 + self.multimodal_config = multimodal_config
420 +
421 + # TODO: Optionally initializes this for supporting embeddings.
422 + self.vision_tower = init_vision_tower_for_MiniMaxVL01(
423 + config,
424 + quant_config,
425 + require_post_norm=False,
426 + prefix=maybe_prefix(prefix, "vision_tower"))
427 + self.multi_modal_projector = MiniMaxVL01MultiModalProjector(
428 + vision_hidden_size=config.vision_config.hidden_size,
429 + text_hidden_size=config.text_config.hidden_size,
430 + projector_hidden_act=config.projector_hidden_act,
431 + multimodal_projector_bias=True,
432 + quant_config=quant_config,
433 + prefix=maybe_prefix(prefix, "multi_modal_projector"))
434 + self.image_newline = nn.Parameter(
435 + torch.empty(config.text_config.hidden_size))
436 + self.language_model = init_vllm_registered_model(
437 + vllm_config=vllm_config,
438 + hf_config=config.text_config,
439 + prefix=maybe_prefix(prefix, "language_model"),
440 + )
441 + self.vision_feature_layer = config.vision_feature_layer
442 + self.vocab_size = config.text_config.vocab_size
443 + self.pad_token_id = -1
444 + if self.config.pad_token_id is not None:
445 + self.pad_token_id = self.config.pad_token_id
446 +
447 + self.make_empty_intermediate_tensors = (
448 + self.language_model.make_empty_intermediate_tensors)
449 +
450 + def get_input_embeddings(
451 + self,
452 + input_ids: torch.Tensor,
453 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
454 + ) -> torch.Tensor:
455 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
456 + if multimodal_embeddings is not None:
457 + inputs_embeds = merge_multimodal_embeddings(
458 + input_ids,
459 + inputs_embeds,
460 + multimodal_embeddings,
461 + self.config.image_token_index,
462 + )
463 + return inputs_embeds
464 +
465 + def get_language_model(self) -> torch.nn.Module:
466 + return self.language_model
467 +
468 + def _select_image_features(self, image_features: torch.Tensor, *,
469 + strategy: str) -> torch.Tensor:
470 + if strategy == "default":
471 + return image_features[:, 1:]
472 + elif strategy == "full":
473 + return image_features
474 +
475 + raise ValueError(f"Unexpected select feature strategy: {strategy}")
476 +
477 + def _image_pixels_to_features(
478 + self,
479 + vision_tower: Union[CLIPVisionModel],
480 + pixel_values: Union[torch.Tensor, list[torch.Tensor]],
481 + ) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]:
482 + # NOTE: we skip the step to select the vision feature layer since
483 + # this is already done inside the vision tower
484 + image_features = vision_tower(pixel_values)
485 +
486 + def select_features(leaf: torch.Tensor):
487 + return self._select_image_features(
488 + leaf,
489 + strategy=self.config.vision_feature_select_strategy,
490 + )
491 +
492 + return cast(
493 + Union[torch.Tensor, tuple[torch.Tensor, ...]],
494 + json_map_leaves(select_features, image_features),
495 + )
496 +
497 + def _process_image_pixels(
498 + self,
499 + inputs: Union[MiniMaxVL01ImagePixelInputs],
500 + ) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]:
501 + assert self.vision_tower is not None
502 +
503 + pixel_values = inputs["pixel_values"]
504 +
505 + return self._image_pixels_to_features(self.vision_tower, pixel_values)
506 +
507 + def _process_image_input(
508 + self,
509 + image_input: MiniMaxVL01ImagePixelInputs,
510 + ) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]:
511 + if image_input["type"] == "image_embeds":
512 + return image_input["data"]
513 +
514 + assert self.vision_tower is not None
515 + image_features = self._process_image_pixels(image_input)
516 +
517 + if isinstance(image_features, torch.Tensor):
518 + return self.multi_modal_projector(image_features)
519 +
520 + feature_sizes = [
521 + image_feature.shape[0] for image_feature in image_features
522 + ]
523 +
524 + image_embeds = self.multi_modal_projector(torch.cat(image_features))
525 + image_embeds = torch.split(image_embeds, feature_sizes)
526 + return image_embeds
527 +
528 + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
529 + h = w = self.config.vision_config.image_size
530 + expected_dims = (3, h, w)
531 + actual_dims = tuple(data.shape[1:])
532 +
533 + if actual_dims != expected_dims:
534 + expected_expr = ("batch_size", *map(str, expected_dims))
535 + raise ValueError(
536 + f"The expected shape of pixel values is {expected_expr}. "
537 + f"You supplied {tuple(data.shape)}.")
538 +
539 + return data
540 +
541 + def _parse_and_validate_image_input(
542 + self, **kwargs: object) -> Optional[MiniMaxVL01ImagePixelInputs]:
543 + pixel_values = kwargs.pop("pixel_values", None)
544 + image_embeds = kwargs.pop("image_embeds", None)
545 +
546 + if pixel_values is None and image_embeds is None:
547 + return None
548 +
549 + if pixel_values is not None:
550 + if not isinstance(pixel_values, (torch.Tensor, list)):
551 + raise ValueError("Incorrect type of pixel values. "
552 + f"Got type: {type(pixel_values)}")
553 +
554 + return MiniMaxVL01ImagePixelInputs(
555 + type="pixel_values",
556 + pixel_values=self._validate_pixel_values(
557 + flatten_bn(pixel_values, concat=True)),
558 + )
559 +
560 + if image_embeds is not None:
561 + if not isinstance(image_embeds, (torch.Tensor, list)):
562 + raise ValueError("Incorrect type of image embeddings. "
563 + f"Got type: {type(image_embeds)}")
564 +
565 + return MiniMaxVL01ImageEmbeddingInputs(
566 + type="image_embeds",
567 + data=flatten_bn(image_embeds, concat=True),
568 + )
569 +
570 + raise AssertionError("This line should be unreachable.")
571 +
572 + def get_multimodal_embeddings(
573 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:
574 + image_input = self._parse_and_validate_image_input(**kwargs)
575 + if image_input is None:
576 + return None
577 +
578 + return self._process_image_input(image_input)
579 +
580 + def forward(
581 + self,
582 + input_ids: torch.Tensor,
583 + positions: torch.Tensor,
584 + intermediate_tensors: Optional[IntermediateTensors] = None,
585 + inputs_embeds: Optional[torch.Tensor] = None,
586 + **kwargs: object,
587 + ) -> Union[torch.Tensor, IntermediateTensors]:
588 +
589 + if intermediate_tensors is not None:
590 + inputs_embeds = None
591 + elif inputs_embeds is None:
592 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
593 + inputs_embeds = self.get_input_embeddings(input_ids,
594 + vision_embeddings)
595 + input_ids = None
596 +
597 + hidden_states = self.language_model.model(input_ids,
598 + positions,
599 + intermediate_tensors,
600 + inputs_embeds=inputs_embeds)
601 +
602 + return hidden_states
603 +
604 + def compute_logits(
605 + self,
606 + hidden_states: torch.Tensor,
607 + sampling_metadata: SamplingMetadata,
608 + ) -> Optional[torch.Tensor]:
609 + return self.language_model.compute_logits(hidden_states,
610 + sampling_metadata)
611 +
612 + def load_weights(self, weights: Iterable[Tuple[str,
613 + torch.Tensor]]) -> Set[str]:
614 + loader = AutoWeightsLoader(self)
615 + return loader.load_weights(weights)
vllm/model_executor/models/registry.py
+1
@@ -189,6 +189,7 @@ _MULTIMODAL_MODELS = {
189 "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"), # noqa: E501
190 "LlavaOnevisionForConditionalGeneration": ("llava_onevision", "LlavaOnevisionForConditionalGeneration"), # noqa: E501
191 "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"), # noqa: E501
192 + "MiniMaxVL01ForConditionalGeneration": ("minimax_vl_01", "MiniMaxVL01ForConditionalGeneration"), # noqa: E501
193 "MiniCPMO": ("minicpmo", "MiniCPMO"),
194 "MiniCPMV": ("minicpmv", "MiniCPMV"),
195 "Mistral3ForConditionalGeneration": ("mistral3", "Mistral3ForConditionalGeneration"), # noqa: E501
vllm/transformers_utils/config.py
+9 -5
@@ -34,11 +34,13 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, Cohere2Config,
34 H2OVLChatConfig,
35 InternVLChatConfig, JAISConfig,
36 KimiVLConfig, MedusaConfig,
37 - MllamaConfig, MLPSpeculatorConfig,
38 - MPTConfig, NemotronConfig,
39 - NVLM_D_Config, RWConfig,
40 - SkyworkR1VChatConfig, SolarConfig,
41 - Telechat2Config, UltravoxConfig)
37 + MiniMaxText01Config,
38 + MiniMaxVL01Config, MllamaConfig,
39 + MLPSpeculatorConfig, MPTConfig,
40 + NemotronConfig, NVLM_D_Config,
41 + RWConfig, SkyworkR1VChatConfig,
42 + SolarConfig, Telechat2Config,
43 + UltravoxConfig)
44 # yapf: enable
45 from vllm.transformers_utils.utils import check_gguf_file
46 from vllm.utils import resolve_obj_by_qualname
@@ -73,6 +75,8 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
75 "exaone": ExaoneConfig,
76 "h2ovl_chat": H2OVLChatConfig,
77 "internvl_chat": InternVLChatConfig,
78 + "minimax_text_01": MiniMaxText01Config,
79 + "minimax_vl_01": MiniMaxVL01Config,
80 "nemotron": NemotronConfig,
81 "NVLM_D": NVLM_D_Config,
82 "solar": SolarConfig,
vllm/transformers_utils/configs/__init__.py
+4
@@ -15,6 +15,8 @@ from vllm.transformers_utils.configs.internvl import InternVLChatConfig
15 from vllm.transformers_utils.configs.jais import JAISConfig
16 from vllm.transformers_utils.configs.kimi_vl import KimiVLConfig
17 from vllm.transformers_utils.configs.medusa import MedusaConfig
18 +from vllm.transformers_utils.configs.minimax_text_01 import MiniMaxText01Config
19 +from vllm.transformers_utils.configs.minimax_vl_01 import MiniMaxVL01Config
20 from vllm.transformers_utils.configs.mllama import MllamaConfig
21 from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
22 from vllm.transformers_utils.configs.moonvit import MoonViTConfig
@@ -39,6 +41,8 @@ __all__ = [
41 "MedusaConfig",
42 "EAGLEConfig",
43 "ExaoneConfig",
44 + "MiniMaxText01Config",
45 + "MiniMaxVL01Config",
46 "MllamaConfig",
47 "MLPSpeculatorConfig",
48 "MoonViTConfig",
vllm/transformers_utils/configs/minimax_text_01.py new
+69
@@ -0,0 +1,69 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +""" MiniMaxText01 model configuration"""
3 +
4 +from transformers.configuration_utils import PretrainedConfig
5 +
6 +
7 +class MiniMaxText01Config(PretrainedConfig):
8 + model_type = "MiniMaxText01"
9 + keys_to_ignore_at_inference = ["past_key_values"]
10 +
11 + def __init__(
12 + self,
13 + vocab_size=32000,
14 + hidden_size=4096,
15 + intermediate_size=14336,
16 + num_hidden_layers=32,
17 + num_attention_heads=32,
18 + num_key_value_heads=8,
19 + hidden_act="silu",
20 + max_position_embeddings=4096 * 32,
21 + initializer_range=0.02,
22 + rms_norm_eps=1e-5,
23 + use_cache=True,
24 + pad_token_id=None,
25 + bos_token_id=None,
26 + eos_token_id=None,
27 + tie_word_embeddings=False,
28 + rope_theta=1e6,
29 + sliding_window=None,
30 + attention_dropout=0.0,
31 + num_experts_per_tok=2,
32 + num_local_experts=8,
33 + output_router_logits=False,
34 + router_aux_loss_coef=0.001,
35 + router_jitter_noise=0.0,
36 + **kwargs,
37 + ):
38 + self.vocab_size = vocab_size
39 + self.max_position_embeddings = max_position_embeddings
40 + self.hidden_size = hidden_size
41 + self.intermediate_size = intermediate_size
42 + self.num_hidden_layers = num_hidden_layers
43 + self.num_attention_heads = num_attention_heads
44 + self.sliding_window = sliding_window
45 +
46 + # for backward compatibility
47 + if num_key_value_heads is None:
48 + num_key_value_heads = num_attention_heads
49 +
50 + self.num_key_value_heads = num_key_value_heads
51 + self.hidden_act = hidden_act
52 + self.initializer_range = initializer_range
53 + self.rms_norm_eps = rms_norm_eps
54 + self.use_cache = use_cache
55 + self.rope_theta = rope_theta
56 + self.attention_dropout = attention_dropout
57 +
58 + self.num_experts_per_tok = num_experts_per_tok
59 + self.num_local_experts = num_local_experts
60 + self.output_router_logits = output_router_logits
61 + self.router_aux_loss_coef = router_aux_loss_coef
62 + self.router_jitter_noise = router_jitter_noise
63 + super().__init__(
64 + pad_token_id=pad_token_id,
65 + bos_token_id=bos_token_id,
66 + eos_token_id=eos_token_id,
67 + tie_word_embeddings=tie_word_embeddings,
68 + **kwargs,
69 + )
vllm/transformers_utils/configs/minimax_vl_01.py new
+70
@@ -0,0 +1,70 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +"""MiniMaxVL01 model configuration"""
3 +
4 +from transformers.configuration_utils import PretrainedConfig
5 +from transformers.models.auto import CONFIG_MAPPING
6 +
7 +from .minimax_text_01 import MiniMaxText01Config
8 +
9 +
10 +class MiniMaxVL01Config(PretrainedConfig):
11 + model_type = "minimax_vl_01"
12 +
13 + def __init__(
14 + self,
15 + vision_config=None,
16 + text_config=None,
17 + ignore_index=-100,
18 + image_token_index=32000,
19 + projector_hidden_act="gelu",
20 + vision_feature_select_strategy="default",
21 + vision_feature_layer=-2,
22 + image_grid_pinpoints=None,
23 + tie_word_embeddings=False,
24 + image_seq_length=576,
25 + **kwargs,
26 + ):
27 + self.ignore_index = ignore_index
28 + self.image_token_index = image_token_index
29 + self.projector_hidden_act = projector_hidden_act
30 + self.image_seq_length = image_seq_length
31 +
32 + if vision_feature_select_strategy not in ["default", "full"]:
33 + raise ValueError("vision_feature_select_strategy should " +
34 + "be one of 'default', 'full'." +
35 + f"Got: {vision_feature_select_strategy}")
36 +
37 + self.vision_feature_select_strategy = vision_feature_select_strategy
38 + self.vision_feature_layer = vision_feature_layer
39 + image_grid_pinpoints = (
40 + image_grid_pinpoints if image_grid_pinpoints is not None else
41 + [[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]])
42 + self.image_grid_pinpoints = image_grid_pinpoints
43 +
44 + if isinstance(vision_config, dict):
45 + if "model_type" not in vision_config:
46 + vision_config["model_type"] = "clip_vision_model"
47 + vision_config = CONFIG_MAPPING[vision_config["model_type"]](
48 + **vision_config)
49 + elif vision_config is None:
50 + vision_config = CONFIG_MAPPING["clip_vision_model"](
51 + intermediate_size=4096,
52 + hidden_size=1024,
53 + patch_size=14,
54 + image_size=336,
55 + num_hidden_layers=24,
56 + num_attention_heads=16,
57 + vocab_size=32000,
58 + projection_dim=768,
59 + )
60 +
61 + self.vision_config = vision_config
62 +
63 + if text_config is not None:
64 + text_config = MiniMaxText01Config(**text_config)
65 + else:
66 + text_config = MiniMaxText01Config()
67 +
68 + self.text_config = text_config
69 +
70 + super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)