A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 03:45 UTC
[MM][Gemma4] Use video profiling hints in encoder budget (#41837)
Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com> Co-authored-by: lesj0610 <lesj0610@users.noreply.github.com>
SeongJun Lee committed
May 7, 2026 at 21:46 UTC
f650ace6de5acdf0616c70282b0b7f426bb52a61
3 files changed
+50
-4
tests/models/multimodal/processing/test_gemma4.py
+35
@@ -1,6 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from collections.abc import Mapping
5
+
6
import pytest
7
from PIL import Image as PILImage
8
@@ -102,6 +104,39 @@ def test_get_mm_max_tokens_per_item_respects_configured_max_soft_tokens(
104
assert tokens["video"] == 32 * (70 + 2 + 6)
105
106
107
+@pytest.mark.parametrize(
108
+ ("limit_mm_per_prompt", "expected_video_tokens"),
109
+ [
110
+ ({"video": 1}, 32 * (70 + 2 + 6)),
111
+ ({"video": {"count": 1}}, 32 * (70 + 2 + 6)),
112
+ ({"video": {"count": 1, "num_frames": 1}}, 1 * (70 + 2 + 6)),
113
+ ({"video": {"count": 1, "num_frames": 8}}, 8 * (70 + 2 + 6)),
114
+ ({"video": {"count": 1, "num_frames": 32}}, 32 * (70 + 2 + 6)),
115
+ ({"video": {"count": 1, "num_frames": 40}}, 32 * (70 + 2 + 6)),
116
+ ],
117
+)
118
+@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
119
+def test_get_mm_max_tokens_per_item_respects_configured_video_num_frames(
120
+ model_id: str,
121
+ limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]],
122
+ expected_video_tokens: int,
123
+):
124
+ ctx = build_model_context(
125
+ model_id,
126
+ limit_mm_per_prompt=limit_mm_per_prompt,
127
+ )
128
+ processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
129
+
130
+ tokens = processor.info.get_mm_max_tokens_per_item(
131
+ seq_len=ctx.model_config.max_model_len,
132
+ mm_counts={"video": 1},
133
+ )
134
+
135
+ assert tokens is not None
136
+ assert tokens["image"] == 280
137
+ assert tokens["video"] == expected_video_tokens
138
+
139
+
140
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
141
def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
142
ctx = build_model_context(
tests/models/utils.py
+6
-3
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import warnings
5
-from collections.abc import Sequence
5
+from collections.abc import Mapping, Sequence
6
from dataclasses import dataclass
7
from typing import Any
8
@@ -277,7 +277,7 @@ def build_model_context(
277
dtype: ModelDType = "auto",
278
model_config_kwargs: dict[str, Any] | None = None,
279
mm_processor_kwargs: dict[str, Any] | None = None,
280
- limit_mm_per_prompt: dict[str, int] | None = None,
280
+ limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]] | None = None,
281
mm_processor_cache_gb: int = 0,
282
):
283
"""Creates an InputProcessingContext for a given model.
@@ -300,7 +300,10 @@ def build_model_context(
300
)
301
302
model_config_kwargs = model_config_kwargs or {}
303
- limit_mm_per_prompt = limit_mm_per_prompt or {}
303
+ limit_mm_per_prompt = {
304
+ modality: dict(limit) if isinstance(limit, Mapping) else limit
305
+ for modality, limit in (limit_mm_per_prompt or {}).items()
306
+ }
307
model_config = ModelConfig(
308
model_id,
309
runner=runner,
vllm/model_executor/models/gemma4_mm.py
+9
-1
@@ -246,7 +246,15 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
246
processor = self.get_hf_processor()
247
tokens["audio"] = processor.audio_seq_length
248
# Video: each frame ≤ 70 soft tokens + boi + eoi + ~6 ts tokens.
249
- tokens["video"] = _VIDEO_MAX_FRAMES * (_VIDEO_MAX_SOFT_TOKENS + 2 + 6)
249
+ num_frames = _VIDEO_MAX_FRAMES
250
+ mm_config = self.ctx.model_config.get_multimodal_config()
251
+ video_opts = mm_config.limit_per_prompt.get("video")
252
+ if (
253
+ isinstance(video_opts, VideoDummyOptions)
254
+ and video_opts.num_frames is not None
255
+ ):
256
+ num_frames = min(num_frames, video_opts.num_frames)
257
+ tokens["video"] = num_frames * (_VIDEO_MAX_SOFT_TOKENS + 2 + 6)
258
return tokens
259
260
def get_data_parser(self) -> MultiModalDataParser: