[BugFix][Multimodal] Pick worst-case DeepSeek-V4 VL dummy image size (#59271) (#59373)
Signed-off-by: Yida Weng <84164537+YidaWeng@users.noreply.github.com> Co-authored-by: Grok <grok@x.ai>
Yida Weng committed
Sep 30, 2026 at 23:09 UTC
17e9295dd565d0a2c2fe7404ecd27705aafbf354
2 files changed
+80
-8
tests/models/multimodal/processing/test_deepseek_v4_vl.py
new
+53
@@ -0,0 +1,53 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""DeepSeek-V4 VL processing helpers."""
4
+
5
+from PIL import Image
6
+
7
+from vllm.models.deepseek_v4.common.mm_preprocess import (
8
+ DeepseekV4VLProcessingInfo,
9
+ DeepseekV4VLProcessor,
10
+ build_image_block_pad_free,
11
+)
12
+from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
13
+
14
+
15
+class _ConfigCtx:
16
+ def __init__(self, config: DeepseekV4Config) -> None:
17
+ self._config = config
18
+
19
+ def get_hf_config(self, *args, **kwargs):
20
+ return self._config
21
+
22
+
23
+def _feature_counts(
24
+ processor: DeepseekV4VLProcessor, width: int, height: int
25
+) -> tuple[int, int]:
26
+ out = processor(images=[Image.new("RGB", (width, height))])
27
+ n_llm_h, n_llm_w = out["llm_grid"][0].tolist()
28
+ types, _ = build_image_block_pad_free(n_llm_h, n_llm_w)
29
+ return len(types), out["patches"].shape[0]
30
+
31
+
32
+def test_get_image_size_with_most_features_beats_wide_real_image():
33
+ """Dummy size must maximize tokens/patches within the budget (#59271).
34
+
35
+ A square under-counts because each aligner row pays an IMAGE_NEW_LINE.
36
+ """
37
+ config = DeepseekV4Config(vision_n_layers=24)
38
+ info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
39
+ processor = DeepseekV4VLProcessor(config)
40
+
41
+ dummy = info.get_image_size_with_most_features()
42
+ dummy_features = _feature_counts(processor, dummy.width, dummy.height)
43
+ wide_features = _feature_counts(processor, 1932, 336)
44
+
45
+ assert dummy == (1932, 336)
46
+ assert wide_features <= dummy_features
47
+
48
+
49
+def test_get_image_size_with_most_features_respects_max_wh_ratio():
50
+ config = DeepseekV4Config(vision_n_layers=24)
51
+ info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
52
+ dummy = info.get_image_size_with_most_features()
53
+ assert dummy.width / dummy.height <= config.vision_max_wh_ratio
vllm/models/deepseek_v4/common/mm_preprocess.py
+27
-8
@@ -354,15 +354,34 @@ class DeepseekV4VLProcessingInfo(BaseProcessingInfo):
354
hf_config = self.get_hf_config()
355
patch_size = hf_config.vision_patch_size
356
downsample_ratio = hf_config.vision_downsample_ratio
357
- # A square maximizes the ViT patch count (area) within the token
358
- # budget; solve the budget-derived size directly to keep the dummy
359
- # image small.
357
+ max_wh_ratio = hf_config.vision_max_wh_ratio
358
+ # Every aligner row costs an extra IMAGE_NEW_LINE token, so height is
359
+ # more expensive than width and a square is *not* optimal: search the
360
+ # LLM grids that fit the budget for the one with the most ViT patches
361
+ # (ties broken by the longer sentinel block).
362
budget = hf_config.vision_max_n_token - (COMPRESS_PAD_TO - 1)
361
- side = budget * patch_size * downsample_ratio
362
- _, _, best_h, best_w, _ = solve_resize_ratio(
363
- side, side, patch_size, downsample_ratio, budget
364
- )
365
- return ImageSize(width=best_w, height=best_h)
363
+ step = patch_size * downsample_ratio
364
+ best_grid = (1, 1)
365
+ best_key = (0, 0)
366
+ for n_llm_h in range(1, budget):
367
+ if (
368
+ grid_tokens(n_llm_h * step, step, patch_size, downsample_ratio)[2]
369
+ > budget
370
+ ):
371
+ break
372
+ for n_llm_w in range(1, budget):
373
+ if max_wh_ratio is not None and n_llm_w > n_llm_h * max_wh_ratio:
374
+ break
375
+ num_tokens = grid_tokens(
376
+ n_llm_h * step, n_llm_w * step, patch_size, downsample_ratio
377
+ )[2]
378
+ if num_tokens > budget:
379
+ break
380
+ key = (n_llm_h * n_llm_w, num_tokens)
381
+ if key > best_key:
382
+ best_key, best_grid = key, (n_llm_h, n_llm_w)
383
+ n_llm_h, n_llm_w = best_grid
384
+ return ImageSize(width=n_llm_w * step, height=n_llm_h * step)
385
386
387
class DeepseekV4VLDummyInputsBuilder(