[Bugfix][GLM-5.3] Size the image encoder cache from the exact token ceiling (#59565)
Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Co-authored-by: Kimi Code <kimi-code@moonshot.cn> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Yufeng He committed
Oct 2, 2026 at 18:43 UTC
5688a4dd4a1ebb5d2d7b6fa863d20cc903f8b341
2 files changed
+75
tests/models/multimodal/processing/test_glm5next.py
+51
@@ -188,3 +188,54 @@ def test_mm_device_do_normalize():
188
torch.testing.assert_close(
189
output, normalized_values.to(device), rtol=1e-5, atol=1e-6
190
)
191
+
192
+
193
+def _image_info(**kwargs):
194
+ ctx = build_model_context(
195
+ "zai-org/GLM-5.3-Flash",
196
+ limit_mm_per_prompt={"image": 1},
197
+ **kwargs,
198
+ )
199
+ return MULTIMODAL_REGISTRY.create_processor(
200
+ ctx.model_config,
201
+ tokenizer=ctx.tokenizer,
202
+ ).info
203
+
204
+
205
+def test_image_encoder_cache_covers_full_token_budget():
206
+ """The most-features probe must reach the processor's token ceiling.
207
+
208
+ The inherited square probe refits to 89x89 = 7921 tokens under the
209
+ max_image_tokens=8000 budget, so the encoder cache came up short and
210
+ ordinary non-square images in 7922-8000 tokens were refused with
211
+ HTTP 400 (#59539).
212
+ """
213
+ info = _image_info()
214
+ assert info.get_max_image_tokens() == 8000
215
+
216
+ # The shapes from the issue, with the token counts the processor
217
+ # actually produces; every one must fit the cache.
218
+ for width, height, expected_tokens in [
219
+ (4032, 3024, 7931), # phone photo, 4:3
220
+ (3840, 2160, 7973), # 4K frame, 16:9
221
+ (3508, 2480, 7950), # A4 at 300 dpi
222
+ (2600, 2400, 7998), # 13:12
223
+ (3000, 3000, 7921), # square worst case before the fix
224
+ ]:
225
+ num_tokens = info.get_num_image_tokens(image_width=width, image_height=height)
226
+ assert num_tokens == expected_tokens
227
+ assert num_tokens <= info.get_max_image_tokens()
228
+
229
+ # The profiling dummy covers the real worst case, not the square one.
230
+ size = info.get_image_size_with_most_features()
231
+ assert size.width * size.height == 2240 * 2800
232
+
233
+
234
+def test_image_encoder_cache_follows_max_pixels_override():
235
+ info = _image_info(mm_processor_kwargs={"max_pixels": 1568 * 100})
236
+ assert info.get_max_image_tokens() == 100
237
+ size = info.get_image_size_with_most_features()
238
+ assert (
239
+ info.get_num_image_tokens(image_width=size.width, image_height=size.height)
240
+ == 100
241
+ )
vllm/models/glm5next/common/multimodal.py
+24
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
"""GLM-5.3-Flash vision tower and multimodal processor."""
4
5
+import math
6
from collections.abc import Mapping
7
from functools import partial
8
from typing import Any
@@ -672,6 +673,29 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
673
return int(override)
674
return self._processor_pixel_budget(self.get_hf_processor().video_processor)[1]
675
676
+ def get_image_size_with_most_features(self) -> ImageSize:
677
+ # The inherited square probe strands budget whenever the token
678
+ # ceiling is not a perfect square: with max_image_tokens=8000 the
679
+ # square refits to 2492x2492 (89x89 = 7921 tokens) while a
680
+ # 2240x2800 canvas reaches 80x100 = 8000, so the encoder cache came
681
+ # up short of the processor's own maximum and refused valid images
682
+ # (#59539). Factor the ceiling exactly instead of probing a square.
683
+ vision_config = self.get_hf_config().vision_config
684
+ factor = (
685
+ vision_config.patch_size
686
+ * vision_config.spatial_merge_size
687
+ * self.get_hf_processor().image_processor.patch_expand_factor
688
+ )
689
+ pixels_per_token = vision_config.temporal_patch_size * factor * factor
690
+ max_tokens = max(1, self._get_image_max_pixels() // pixels_per_token)
691
+ short_side = math.isqrt(max_tokens)
692
+ while max_tokens % short_side:
693
+ short_side -= 1
694
+ return ImageSize(
695
+ width=(max_tokens // short_side) * factor,
696
+ height=short_side * factor,
697
+ )
698
+
699
def _get_video_second_idx_glm46v(
700
self, metadata: dict[str, Any], total_frames: int
701
) -> list[int]: