A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
Fix PixtralHF missing spatial_merge_size (#17571)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
May 1, 2025 at 23:14 UTC
f192ca90e6e8ab7b1b0015040e521c5374f5c812
4 files changed
+18
-25
vllm/model_executor/models/llava.py
+2
-3
@@ -354,9 +354,8 @@ class PixtralHFMultiModalProcessor(
354
image_token_id = hf_config.image_token_index
355
image_end_id = vocab[processor.image_end_token]
356
357
- vision_config = hf_config.vision_config
358
- assert isinstance(vision_config, PixtralVisionConfig)
359
- encoder_info = PixtralHFEncoderInfo(vision_config)
357
+ assert isinstance(hf_config.vision_config, PixtralVisionConfig)
358
+ encoder_info = PixtralHFEncoderInfo(hf_config)
359
360
def get_replacement(item_idx: int):
361
images = mm_items.get_items("image", ImageProcessorItems)
vllm/model_executor/models/mistral3.py
+2
-6
@@ -272,12 +272,8 @@ class Mistral3MultiModalProcessor(
272
image_token_id = hf_config.image_token_index
273
image_end_id = vocab[processor.image_end_token]
274
275
- vision_config = hf_config.vision_config
276
- assert isinstance(vision_config, PixtralVisionConfig)
277
- # Need to sneak in spatial_merge_size for Mistral3
278
- vision_config.spatial_merge_size = getattr(hf_config,
279
- "spatial_merge_size", 1)
280
- encoder_info = PixtralHFEncoderInfo(vision_config)
275
+ assert isinstance(hf_config.vision_config, PixtralVisionConfig)
276
+ encoder_info = PixtralHFEncoderInfo(hf_config)
277
278
def get_replacement(item_idx: int):
279
images = mm_items.get_items("image", ImageProcessorItems)
vllm/model_executor/models/pixtral.py
+3
-2
@@ -916,8 +916,9 @@ class PixtralHFEncoderInfo(VisionEncoderInfo[PixtralVisionConfig]):
916
return self.vision_config.image_size
917
918
def get_patch_size(self) -> int:
919
- return (self.vision_config.patch_size *
920
- self.vision_config.spatial_merge_size)
919
+ # spatial_merge_size is needed for Mistral3
920
+ spatial_merge_size = getattr(self.hf_config, "spatial_merge_size", 1)
921
+ return self.vision_config.patch_size * spatial_merge_size
922
923
def get_patch_grid_length(self) -> int:
924
image_size, patch_size = self.get_image_size(), self.get_patch_size()
vllm/model_executor/models/vision.py
+11
-14
@@ -19,10 +19,11 @@ _C = TypeVar("_C", bound=PretrainedConfig)
19
20
class VisionEncoderInfo(ABC, Generic[_C]):
21
22
- def __init__(self, vision_config: _C) -> None:
22
+ def __init__(self, hf_config: _C) -> None:
23
super().__init__()
24
25
- self.vision_config = vision_config
25
+ self.hf_config = hf_config
26
+ self.vision_config = hf_config.vision_config
27
28
@abstractmethod
29
def get_num_image_tokens(
@@ -57,18 +58,14 @@ def get_vision_encoder_info(
58
from .pixtral import PixtralHFEncoderInfo, PixtralVisionConfig
59
from .siglip import SiglipEncoderInfo, SiglipVisionConfig
60
60
- vision_config = hf_config.vision_config
61
- if isinstance(vision_config, CLIPVisionConfig):
62
- return CLIPEncoderInfo(vision_config)
63
- if isinstance(vision_config, PixtralVisionConfig):
64
- # Need to sneak in spatial_merge_size for Mistral3
65
- vision_config.spatial_merge_size = getattr(hf_config,
66
- "spatial_merge_size", 1)
67
- return PixtralHFEncoderInfo(vision_config)
68
- if isinstance(vision_config, SiglipVisionConfig):
69
- return SiglipEncoderInfo(vision_config)
70
-
71
- msg = f"Unsupported vision config: {type(vision_config)}"
61
+ if isinstance(hf_config.vision_config, CLIPVisionConfig):
62
+ return CLIPEncoderInfo(hf_config)
63
+ if isinstance(hf_config.vision_config, PixtralVisionConfig):
64
+ return PixtralHFEncoderInfo(hf_config)
65
+ if isinstance(hf_config.vision_config, SiglipVisionConfig):
66
+ return SiglipEncoderInfo(hf_config)
67
+
68
+ msg = f"Unsupported vision config: {type(hf_config.vision_config)}"
69
raise NotImplementedError(msg)
70
71