[Bugfix][Model] Mistral3: align placeholder grid with public processor (#53758)
Signed-off-by: Oliver Holworthy <1216955+oliverholworthy@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Oliver Holworthy committed
Sep 21, 2026 at 07:36 UTC
9598487bc1829504c8af81d6007ab6b9deeaa15a
2 files changed
+54
-37
tests/models/multimodal/processing/test_mistral3.py
+49
-27
@@ -5,7 +5,7 @@
5
import pytest
6
import torch
7
from PIL import Image
8
-from transformers import BatchFeature
8
+from transformers import AutoProcessor, BatchFeature
9
10
from vllm.model_executor.models.lightonocr import LightOnOCRProcessingInfo
11
from vllm.model_executor.models.mistral3 import Mistral3HFEncoderInfo
@@ -22,25 +22,41 @@ _MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
22
_LIGHTON_MODEL_ID = "lightonai/LightOnOCR-1B-1025"
23
24
25
-def _processed_pixel_values(
25
+def _process_images_with_hf(
26
hf_processor,
27
images: list[Image.Image],
28
mm_processor_kwargs: dict[str, object],
29
-) -> list[torch.Tensor]:
30
- """Resize via the HF image processor and un-pad to per-image H×W."""
31
- image_processor = hf_processor.image_processor
32
- hf_out = image_processor(images=images, return_tensors="pt", **mm_processor_kwargs)
29
+) -> tuple[list[torch.Tensor], list[int]]:
30
+ """Process images and placeholders through the public HF processor."""
31
+ hf_out = hf_processor(
32
+ text=hf_processor.image_token * len(images),
33
+ images=images,
34
+ return_tensors="pt",
35
+ **mm_processor_kwargs,
36
+ )
37
pixel_values = hf_out["pixel_values"]
38
image_sizes = hf_out["image_sizes"]
35
- return [p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)]
36
-
37
-
38
-def _placeholder_count_from_prompt_updates(
39
+ unpadded = [p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)]
40
+
41
+ special_token_ids = {
42
+ hf_processor.image_token_id,
43
+ hf_processor.image_break_token_id,
44
+ hf_processor.image_end_token_id,
45
+ }
46
+ placeholder_tokens = [
47
+ token_id
48
+ for token_id in hf_out["input_ids"][0].tolist()
49
+ if token_id in special_token_ids
50
+ ]
51
+ return unpadded, placeholder_tokens
52
+
53
+
54
+def _placeholder_tokens_from_prompt_updates(
55
processor,
56
images: list[Image.Image],
57
pixel_values: list[torch.Tensor],
58
mm_processor_kwargs: dict[str, object],
43
-) -> int:
59
+) -> list[int]:
60
hf_inputs = BatchFeature({"pixel_values": pixel_values})
61
fields_config = processor._get_mm_fields_config(hf_inputs, mm_processor_kwargs)
62
out_mm_kwargs = MultiModalKwargsItems.from_hf_inputs(hf_inputs, fields_config)
@@ -49,13 +65,11 @@ def _placeholder_count_from_prompt_updates(
65
updates = processor._get_prompt_updates(
66
mm_items, mm_processor_kwargs, out_mm_kwargs
67
)
52
- image_token_id = processor.info.get_hf_config().image_token_index
53
-
54
- total = 0
68
+ placeholder_tokens: list[int] = []
69
for item_idx in range(len(images)):
70
details = updates[0].resolve(item_idx).content
57
- total += details.full.count(image_token_id)
58
- return total
71
+ placeholder_tokens.extend(details.full)
72
+ return placeholder_tokens
73
74
75
def _expected_placeholder_tokens_per_image(
@@ -72,12 +86,12 @@ def _expected_placeholder_tokens_per_image(
86
patch_h = patch_w = int(patch_size)
87
88
spatial_merge_size = getattr(hf_processor, "spatial_merge_size", 1)
75
- assert image_h % patch_h == 0
76
- assert image_w % patch_w == 0
89
+ merged_patch_h = patch_h * spatial_merge_size
90
+ merged_patch_w = patch_w * spatial_merge_size
91
+ assert image_h % merged_patch_h == 0
92
+ assert image_w % merged_patch_w == 0
93
78
- return (image_h // (patch_h * spatial_merge_size)) * (
79
- image_w // (patch_w * spatial_merge_size)
80
- )
94
+ return (image_h // merged_patch_h) * (image_w // merged_patch_w)
95
96
97
@pytest.mark.parametrize("model_id", [_MODEL_ID])
@@ -87,8 +101,8 @@ def _expected_placeholder_tokens_per_image(
101
({}, (448, 448), 256),
102
({"size": {"longest_edge": 1008}}, (1540, 1540), 1296),
103
({"size": {"longest_edge": 1288}}, (1536, 1187), 1656),
90
- ({"size": {"longest_edge": 1008}}, (29, 29), 1),
91
- ({"size": {"longest_edge": 1000}}, (1540, 1700), 1152),
104
+ ({"size": {"longest_edge": 1008}}, (29, 29), 4),
105
+ ({"size": {"longest_edge": 1000}}, (1540, 1700), 1188),
106
],
107
)
108
@pytest.mark.parametrize("num_imgs", [1, 2])
@@ -109,21 +123,29 @@ def test_processor_size_override(
123
)
124
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
125
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
112
- hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
126
+ hf_processor = AutoProcessor.from_pretrained(
127
+ model_id,
128
+ fix_mistral_regex=True,
129
+ )
130
131
dummy_image = Image.new("RGB", image_size, color=(127, 127, 127))
132
images = [dummy_image] * num_imgs
133
merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
117
- pixel_values = _processed_pixel_values(hf_processor, images, merged_mm_kwargs)
134
+ pixel_values, hf_placeholder_tokens = _process_images_with_hf(
135
+ hf_processor, images, merged_mm_kwargs
136
+ )
137
119
- image_token_count = _placeholder_count_from_prompt_updates(
138
+ prompt_update_tokens = _placeholder_tokens_from_prompt_updates(
139
processor, images, pixel_values, hf_processor_mm_kwargs
140
)
141
expected_from_pixel_values = _expected_placeholder_tokens_per_image(
142
hf_processor, pixel_values[0]
143
)
144
assert expected_from_pixel_values == expected_toks_per_img
126
- assert image_token_count == expected_from_pixel_values * num_imgs
145
+ assert hf_placeholder_tokens.count(hf_processor.image_token_id) == (
146
+ expected_from_pixel_values * num_imgs
147
+ )
148
+ assert prompt_update_tokens == hf_placeholder_tokens
149
150
151
def test_lightonocr_keeps_vision_config_image_size():
vllm/model_executor/models/mistral3.py
+5
-10
@@ -192,18 +192,13 @@ class Mistral3HFEncoderInfo(PixtralHFEncoderInfo):
192
image_width = math.floor(image_width / ratio)
193
image_height = math.floor(image_height / ratio)
194
195
- patch_size = self.vision_config.patch_size
195
assert isinstance(self.hf_config, Mistral3Config)
197
- spatial_merge_size = self.hf_config.spatial_merge_size
198
-
199
- # The HF processor rounds each dimension up to the vision patch size
200
- # before the projector drops incomplete spatial-merge groups. This is
201
- # not equivalent to rounding directly to the merged patch size.
202
- num_width_patches = (image_width - 1) // patch_size + 1
203
- num_height_patches = (image_height - 1) // patch_size + 1
196
+ merged_patch_size = (
197
+ self.vision_config.patch_size * self.hf_config.spatial_merge_size
198
+ )
199
205
- ncols = num_width_patches // spatial_merge_size
206
- nrows = num_height_patches // spatial_merge_size
200
+ ncols = (image_width - 1) // merged_patch_size + 1
201
+ nrows = (image_height - 1) // merged_patch_size + 1
202
return ncols, nrows
203
204