A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[Bugfix] Clean up MiniMax-VL and fix processing (#17354)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Apr 29, 2025 at 20:42 UTC
00ee37efa23600d7c89d8fd5dc8bdc125c49e39d
4 files changed
+38
-283
docs/source/models/supported_models.md
+7
@@ -979,6 +979,13 @@ See [this page](#generative-models) for more information on how to use generativ
979
* ✅︎
980
* ✅︎
981
* ✅︎
982
+- * `MiniMaxVL01ForConditionalGeneration`
983
+ * MiniMax-VL
984
+ * T + I<sup>E+</sup>
985
+ * `MiniMaxAI/MiniMax-VL-01`, etc.
986
+ *
987
+ * ✅︎
988
+ * ✅︎
989
- * `Mistral3ForConditionalGeneration`
990
* Mistral3
991
* T + I<sup>+</sup>
tests/models/multimodal/processing/test_common.py
+1
@@ -270,6 +270,7 @@ def _test_processing_correctness_mistral(
270
"openbmb/MiniCPM-Llama3-V-2_5",
271
"openbmb/MiniCPM-o-2_6",
272
"openbmb/MiniCPM-V-2_6",
273
+ "MiniMaxAI/MiniMax-VL-01",
274
"allenai/Molmo-7B-D-0924",
275
"allenai/Molmo-7B-O-0924",
276
"nvidia/NVLM-D-72B",
tests/models/multimodal/processing/test_minimax_vl_01.py
-1
@@ -12,7 +12,6 @@ from ...utils import build_model_context
12
13
14
@pytest.mark.parametrize("model_id", ["MiniMaxAI/MiniMax-VL-01"])
15
-# yapf: enable
15
@pytest.mark.parametrize("num_imgs", [1, 2])
16
def test_processor_override(
17
image_assets: _ImageAssets,
vllm/model_executor/models/minimax_vl_01.py
+30
-282
@@ -1,52 +1,32 @@
1
# SPDX-License-Identifier: Apache-2.0
2
+from collections.abc import Iterable, Mapping
3
+from typing import Literal, Optional, Set, Tuple, TypedDict, Union, cast
4
3
-from abc import abstractmethod
4
-from collections.abc import Iterable, Mapping, Sequence
5
-from dataclasses import dataclass
6
-from typing import (Final, Literal, Optional, Protocol, Set, Tuple, TypedDict,
7
- TypeVar, Union, cast)
8
-
9
-import numpy as np
5
import torch
6
import torch.nn as nn
12
-from transformers import BatchFeature, CLIPVisionConfig, PretrainedConfig
13
-from transformers.image_processing_utils import select_best_resolution
7
+from transformers import BatchFeature
8
9
from vllm.config import VllmConfig
10
from vllm.jsontree import json_map_leaves
17
-from vllm.logger import init_logger
11
from vllm.model_executor.layers.activation import get_act_fn
12
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
13
RowParallelLinear)
14
from vllm.model_executor.layers.quantization import QuantizationConfig
15
from vllm.model_executor.sampling_metadata import SamplingMetadata
23
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
24
-from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
25
-from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
26
- ImageSize, MultiModalDataItems)
27
-from vllm.multimodal.processing import (BaseMultiModalProcessor,
28
- BaseProcessingInfo, PromptReplacement,
29
- PromptUpdate)
30
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
16
+from vllm.multimodal import MULTIMODAL_REGISTRY
17
+from vllm.multimodal.inputs import MultiModalFieldConfig
18
from vllm.sequence import IntermediateTensors
19
from vllm.transformers_utils.configs.minimax_vl_01 import MiniMaxVL01Config
20
21
from .clip import CLIPVisionModel
22
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
23
+from .llava import (BaseLlavaMultiModalProcessor, LlavaDummyInputsBuilder,
24
+ init_vision_tower_for_llava)
25
+from .llava_next import LlavaNextProcessingInfo
26
from .pixtral import PixtralHFVisionModel
27
from .siglip import SiglipVisionModel
28
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
29
maybe_prefix, merge_multimodal_embeddings)
40
-from .vision import get_vision_encoder_info
41
-
42
-logger = init_logger(__name__)
43
-
44
-
45
-# For dummy input only
46
-@dataclass
47
-class MaxImageTokenMeta:
48
- width: int = 1024
49
- height: int = 1024
30
31
32
class MiniMaxVL01ImagePixelInputs(TypedDict):
@@ -69,66 +49,8 @@ class MiniMaxVL01ImageEmbeddingInputs(TypedDict):
49
"""
50
51
72
-def image_size_to_num_patches(image_size, grid_pinpoints, patch_size: int):
73
- if not isinstance(grid_pinpoints, list):
74
- raise TypeError("grid_pinpoints should be a list of tuples or lists")
75
-
76
- # ! VERY IMPORTANT if image_size is tensor, must convert to into tuple,
77
- # otherwise it will cause wrong calculate
78
- if not isinstance(image_size, (list, tuple)):
79
- if not isinstance(image_size, (torch.Tensor, np.ndarray)):
80
- raise TypeError("image_size invalid type " +
81
- f"{type(image_size)} with value {image_size}")
82
- image_size = image_size.tolist()
83
-
84
- best_resolution = select_best_resolution(image_size, grid_pinpoints)
85
- height, width = best_resolution
86
- num_patches = 0
87
- # consider change to ceil(height/patch_size)*ceil(width/patch_size) + 1
88
- for i in range(0, height, patch_size):
89
- for j in range(0, width, patch_size):
90
- num_patches += 1
91
- # add the base patch
92
- num_patches += 1
93
- return num_patches
94
-
95
-
96
-def get_anyres_image_grid_shape(image_size, grid_pinpoints, patch_size):
97
- if not isinstance(grid_pinpoints, list):
98
- raise TypeError("grid_pinpoints should be a list of tuples or lists")
99
-
100
- # ! VERY IMPORTANT if image_size is tensor,
101
- # must convert to into tuple,
102
- # otherwise it will cause wrong calculate
103
- if not isinstance(image_size, (list, tuple)):
104
- if not isinstance(image_size, (torch.Tensor, np.ndarray)):
105
- raise TypeError(
106
- "image_size invalid type " +
107
- f"{type(image_size)} not valid, " +
108
- "should be either list, tuple, np.ndarray or tensor")
109
- image_size = image_size.tolist()
110
-
111
- height, width = select_best_resolution(image_size, grid_pinpoints)
112
- return height // patch_size, width // patch_size
113
-
114
-
115
-def unpad_image(tensor, original_size):
116
- original_height, original_width = original_size
117
- current_height, current_width = tensor.shape[1:]
118
-
119
- original_aspect_ratio = original_width / original_height
120
- current_aspect_ratio = current_width / current_height
121
-
122
- if original_aspect_ratio > current_aspect_ratio:
123
- new_height = int(original_height * current_width) // original_width
124
- padding = (current_height - new_height) // 2
125
- unpadded_tensor = tensor[:, padding:current_height - padding, :]
126
- else:
127
- new_width = int(original_width * current_height) // original_height
128
- padding = (current_width - new_width) // 2
129
- unpadded_tensor = tensor[:, :, padding:current_width - padding]
130
-
131
- return unpadded_tensor
52
+MiniMaxVL01ImageInputs = Union[MiniMaxVL01ImagePixelInputs,
53
+ MiniMaxVL01ImageEmbeddingInputs]
54
55
56
class MiniMaxVL01MultiModalProjector(nn.Module):
@@ -161,144 +83,29 @@ class MiniMaxVL01MultiModalProjector(nn.Module):
83
return hidden_states
84
85
164
-class MiniMaxVL01LikeConfig(Protocol):
165
- vision_config: Final[PretrainedConfig]
166
- image_token_index: Final[int]
167
- vision_feature_select_strategy: Final[str]
168
- vision_feature_layer: Final[Union[int, list[int]]]
169
-
170
-
171
-class MiniMaxVL01LikeProcessor(Protocol):
172
- image_token: Final[str]
173
-
86
+class MiniMaxVL01DummyInputsBuilder(LlavaDummyInputsBuilder):
87
+ pass
88
175
-_I = TypeVar("_I", bound=BaseProcessingInfo)
89
177
-
178
-class MiniMaxVL01DummyInputsBuilder(BaseDummyInputsBuilder[_I]):
179
-
180
- def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
181
- num_images = mm_counts.get("image", 0)
182
- processor = self.info.get_hf_processor()
183
- image_token = processor.image_token
184
- return image_token * num_images
185
-
186
- def get_dummy_mm_data(
187
- self,
188
- seq_len: int,
189
- mm_counts: Mapping[str, int],
190
- ) -> MultiModalDataDict:
191
- num_images = mm_counts.get("image", 0)
192
-
193
- return {
194
- "image":
195
- self._get_dummy_images(width=MaxImageTokenMeta.width,
196
- height=MaxImageTokenMeta.height,
197
- num_images=num_images)
198
- }
199
-
200
-
201
-class MiniMaxVL01ProcessingInfo(BaseProcessingInfo):
90
+class MiniMaxVL01ProcessingInfo(LlavaNextProcessingInfo):
91
92
def get_hf_config(self):
93
return self.ctx.get_hf_config(MiniMaxVL01Config)
94
206
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
207
- return {"image": None}
208
-
209
- def get_vision_encoder_info(self):
210
- return get_vision_encoder_info(self.get_hf_config())
95
+ def get_hf_processor(self, **kwargs: object):
96
+ hf_processor = self.ctx.get_hf_processor(**kwargs)
97
+ image_processor = hf_processor.image_processor
98
+ image_processor.anyres_preprocess = (
99
+ image_processor.anyres_for_vllm_preprocess)
100
212
- def _apply_feature_select_strategy(
213
- self,
214
- strategy: str,
215
- encoder_num_image_tokens: int,
216
- ) -> int:
217
- if strategy == "default":
218
- return encoder_num_image_tokens - 1
219
- if strategy == "full":
220
- return encoder_num_image_tokens
221
-
222
- msg = f"Unexpected feature select strategy: {strategy!r}"
223
- raise NotImplementedError(msg)
224
-
225
- def get_num_image_tokens(
226
- self,
227
- *,
228
- image_width: int,
229
- image_height: int,
230
- ) -> int:
231
- hf_config = self.get_hf_config()
232
- vision_encoder_info = self.get_vision_encoder_info()
233
-
234
- return self._apply_feature_select_strategy(
235
- hf_config.vision_feature_select_strategy,
236
- vision_encoder_info.get_num_image_tokens(
237
- image_width=image_width,
238
- image_height=image_height,
239
- ),
240
- )
101
+ return hf_processor
102
242
- def get_image_size_with_most_features(self) -> ImageSize:
243
- vision_encoder_info = self.get_vision_encoder_info()
244
- width = height = vision_encoder_info.get_image_size()
245
- return ImageSize(width=width, height=height)
246
-
247
- def get_max_image_tokens(self) -> int:
248
- target_width, target_height = self.get_image_size_with_most_features()
249
-
250
- return self.get_num_image_tokens(
251
- image_width=target_width,
252
- image_height=target_height,
253
- )
254
-
255
-
256
-class BaseMiniMaxVL01MultiModalProcessor(BaseMultiModalProcessor[_I]):
257
-
258
- # Copied from BaseMultiModalProcessor
259
- @abstractmethod
260
- def _get_mm_fields_config(
261
- self,
262
- hf_inputs: BatchFeature,
263
- hf_processor_mm_kwargs: Mapping[str, object],
264
- ) -> Mapping[str, MultiModalFieldConfig]:
265
- raise NotImplementedError
266
-
267
- def _get_prompt_updates(
268
- self,
269
- mm_items: MultiModalDataItems,
270
- hf_processor_mm_kwargs: Mapping[str, object],
271
- out_mm_kwargs: MultiModalKwargs,
272
- ) -> Sequence[PromptUpdate]:
273
- hf_config = self.info.get_hf_config()
274
- image_token_id = hf_config.image_token_index
275
-
276
- def get_replacement(item_idx: int):
277
- images = mm_items.get_items(
278
- "image", (ImageEmbeddingItems, ImageProcessorItems))
279
-
280
- if isinstance(images, ImageEmbeddingItems):
281
- num_image_tokens = images.get_feature_size(item_idx)
282
- else:
283
- image_size = images.get_image_size(item_idx)
284
- num_image_tokens = self.info.get_num_image_tokens(
285
- image_width=image_size.width,
286
- image_height=image_size.height,
287
- )
288
-
289
- return [image_token_id] * num_image_tokens
290
-
291
- return [
292
- PromptReplacement(
293
- modality="image",
294
- target=[image_token_id],
295
- replacement=get_replacement,
296
- ),
297
- ]
103
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
104
+ return {"image": None}
105
106
107
class MiniMaxVL01MultiModalProcessor(
301
- BaseMiniMaxVL01MultiModalProcessor[MiniMaxVL01ProcessingInfo]):
108
+ BaseLlavaMultiModalProcessor[MiniMaxVL01ProcessingInfo]):
109
110
def _call_hf_processor(
111
self,
@@ -314,10 +121,9 @@ class MiniMaxVL01MultiModalProcessor(
121
122
pixel_values = processed_outputs.get("pixel_values")
123
if pixel_values is not None:
124
+ # Avoid padding since we need the output for each image to be
125
+ # independent of other images for the cache to work correctly
126
image_sizes = processed_outputs["image_sizes"]
318
- min_len = min(len(pixel_values), len(image_sizes))
319
- pixel_values = pixel_values[:min_len]
320
- image_sizes = image_sizes[:min_len]
127
assert len(pixel_values) == len(image_sizes)
128
129
processed_outputs["pixel_values"] = [
@@ -337,65 +143,6 @@ class MiniMaxVL01MultiModalProcessor(
143
}
144
145
340
-def _get_num_hidden_layers(hf_config: MiniMaxVL01LikeConfig) -> int:
341
- """Determine the number of hidden layers to initialize up to in the
342
- visual encoder.
343
-
344
- Args:
345
- hf_config: Model config with vision feature layer(s).
346
- """
347
- feature_layers = hf_config.vision_feature_layer
348
- num_hidden_layers = hf_config.vision_config.num_hidden_layers
349
- # If we have one feature layer, initialize up to that layer
350
- if isinstance(feature_layers, int):
351
- return _get_layer_index(feature_layers, num_hidden_layers)
352
- # If we have multiple feature layers, initialize up to the deepest one
353
- elif isinstance(feature_layers, (list, tuple)):
354
- return max(
355
- _get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
356
- raise TypeError(f"vision_layer_feature type: {type(feature_layers)}"
357
- " is not supported")
358
-
359
-
360
-def _get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int:
361
- """Given a signed vision feature layer, get the number of hidden layers
362
- needed to leverage it.
363
-
364
- Args:
365
- feature_layer_index: Index of a required layer in the visual encoder.
366
- num_hidden_layers: The total number of hidden layers in the visual
367
- encoder.
368
- """
369
- if feature_layer_index < 0:
370
- return num_hidden_layers + feature_layer_index + 1
371
- return feature_layer_index
372
-
373
-
374
-def init_vision_tower_for_MiniMaxVL01(
375
- hf_config: MiniMaxVL01LikeConfig,
376
- quant_config: Optional[QuantizationConfig],
377
- *,
378
- require_post_norm: Optional[bool] = None,
379
- prefix: str = "",
380
-) -> Union[CLIPVisionModel, SiglipVisionModel, PixtralHFVisionModel]:
381
- vision_config = hf_config.vision_config
382
-
383
- # Initialize the vision tower only up to the deepest required feature layer
384
- num_hidden_layers = _get_num_hidden_layers(hf_config)
385
-
386
- if isinstance(vision_config, CLIPVisionConfig):
387
- return CLIPVisionModel(
388
- vision_config,
389
- quant_config=quant_config,
390
- num_hidden_layers_override=num_hidden_layers,
391
- require_post_norm=require_post_norm,
392
- prefix=prefix,
393
- )
394
-
395
- msg = f"Unsupported vision config: {type(vision_config)}"
396
- raise NotImplementedError(msg)
397
-
398
-
146
@MULTIMODAL_REGISTRY.register_processor(
147
MiniMaxVL01MultiModalProcessor,
148
info=MiniMaxVL01ProcessingInfo,
@@ -419,7 +166,7 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal,
166
self.multimodal_config = multimodal_config
167
168
# TODO: Optionally initializes this for supporting embeddings.
422
- self.vision_tower = init_vision_tower_for_MiniMaxVL01(
169
+ self.vision_tower = init_vision_tower_for_llava(
170
config,
171
quant_config,
172
require_post_norm=False,
@@ -476,7 +223,8 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal,
223
224
def _image_pixels_to_features(
225
self,
479
- vision_tower: Union[CLIPVisionModel],
226
+ vision_tower: Union[CLIPVisionModel, SiglipVisionModel,
227
+ PixtralHFVisionModel],
228
pixel_values: Union[torch.Tensor, list[torch.Tensor]],
229
) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]:
230
# NOTE: we skip the step to select the vision feature layer since
@@ -496,7 +244,7 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal,
244
245
def _process_image_pixels(
246
self,
499
- inputs: Union[MiniMaxVL01ImagePixelInputs],
247
+ inputs: MiniMaxVL01ImagePixelInputs,
248
) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]:
249
assert self.vision_tower is not None
250
@@ -506,7 +254,7 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal,
254
255
def _process_image_input(
256
self,
509
- image_input: MiniMaxVL01ImagePixelInputs,
257
+ image_input: MiniMaxVL01ImageInputs,
258
) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]:
259
if image_input["type"] == "image_embeds":
260
return image_input["data"]
@@ -539,7 +287,7 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal,
287
return data
288
289
def _parse_and_validate_image_input(
542
- self, **kwargs: object) -> Optional[MiniMaxVL01ImagePixelInputs]:
290
+ self, **kwargs: object) -> Optional[MiniMaxVL01ImageInputs]:
291
pixel_values = kwargs.pop("pixel_values", None)
292
image_embeds = kwargs.pop("image_embeds", None)
293