[Bugfix][Gemma4] Keep image kwargs out of video preprocessing (#56652)
Signed-off-by: Ylean <61880605+Woolgathererer@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com>
Ylean committed
Sep 13, 2026 at 17:07 UTC
a987777755c83863b5d563c15630dd324033d53f
2 files changed
+71
tests/models/multimodal/processing/test_gemma4.py
+68
@@ -3,6 +3,7 @@
3
4
from collections.abc import Mapping
5
6
+import numpy as np
7
import pytest
8
import torch
9
from PIL import Image as PILImage
@@ -13,6 +14,7 @@ from vllm.model_executor.models.gemma4_mm import (
14
Gemma4ImagePixelInputs,
15
)
16
from vllm.multimodal import MULTIMODAL_REGISTRY
17
+from vllm.multimodal.cache import MultiModalProcessorOnlyCache
18
from vllm.multimodal.inputs import MultiModalFieldConfig
19
from vllm.utils.mem_constants import GiB_bytes
20
@@ -199,6 +201,72 @@ def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
201
assert replacement == expected
202
203
204
+@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
205
+@pytest.mark.parametrize("kwargs_on_init", [False, True])
206
+@pytest.mark.parametrize(
207
+ "image_kwargs", [{"rescale_factor": 1 / 127.5}, {"max_soft_tokens": 560}]
208
+)
209
+@pytest.mark.parametrize("video_uuid", [None, "same-video"])
210
+def test_video_cache_is_independent_of_image_kwargs(
211
+ model_id: str,
212
+ kwargs_on_init: bool,
213
+ image_kwargs: dict[str, object],
214
+ video_uuid: str | None,
215
+):
216
+ """Image overrides must not change video frames or depend on cache warmth."""
217
+ kwargs = {"images_kwargs": image_kwargs}
218
+ ctx = build_model_context(
219
+ model_id,
220
+ limit_mm_per_prompt={"image": 1, "video": 1},
221
+ mm_processor_cache_gb=1,
222
+ )
223
+ cache = MultiModalProcessorOnlyCache(ctx.model_config)
224
+ processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=cache)
225
+ hf_processor = processor.info.get_hf_processor()
226
+ image = PILImage.new("RGB", (48, 48), color=(128, 128, 128))
227
+ frames = np.stack([np.asarray(image)] * 2)
228
+ metadata = {"fps": 2.0, "frames_indices": [0, 1]}
229
+ mm_items = processor.info.parse_mm_data(
230
+ {"image": image, "video": [(frames, metadata)]}
231
+ )
232
+
233
+ def process(mm_kwargs):
234
+ return processor(
235
+ hf_processor.image_token + hf_processor.video_token,
236
+ mm_items,
237
+ mm_uuid_items={"video": [video_uuid]},
238
+ hf_processor_mm_kwargs=mm_kwargs,
239
+ )
240
+
241
+ baseline = process({})
242
+ if kwargs_on_init:
243
+ ctx = build_model_context(
244
+ model_id,
245
+ mm_processor_kwargs=kwargs,
246
+ limit_mm_per_prompt={"image": 1, "video": 1},
247
+ mm_processor_cache_gb=1,
248
+ )
249
+ cache = MultiModalProcessorOnlyCache(ctx.model_config)
250
+ processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=cache)
251
+ request_kwargs = {} if kwargs_on_init else kwargs
252
+ process(request_kwargs)
253
+ cached = process(request_kwargs)
254
+ cache.clear_cache()
255
+ fresh = process(request_kwargs)
256
+
257
+ def pixels(result, modality, field):
258
+ return result["mm_kwargs"][modality][0][field].data
259
+
260
+ video_pixels = pixels(fresh, "video", "pixel_values_videos")
261
+ assert torch.equal(pixels(cached, "video", "pixel_values_videos"), video_pixels)
262
+ assert torch.equal(pixels(baseline, "video", "pixel_values_videos"), video_pixels)
263
+ assert baseline["mm_hashes"]["video"] == fresh["mm_hashes"]["video"]
264
+ assert not torch.equal(
265
+ pixels(baseline, "image", "pixel_values"),
266
+ pixels(fresh, "image", "pixel_values"),
267
+ )
268
+
269
+
270
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
271
def test_limit_mm_per_prompt(
272
image_assets: ImageTestAssets,
vllm/model_executor/models/gemma4_mm.py
+3
@@ -629,6 +629,9 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
629
630
# Process frames as images with max_soft_tokens=70
631
video_mm_kwargs = dict(hf_processor_mm_kwargs)
632
+ # Override configured image options too: these inputs are video
633
+ # frames, whose cache keys exclude images_kwargs.
634
+ video_mm_kwargs["images_kwargs"] = {}
635
video_mm_kwargs["max_soft_tokens"] = _VIDEO_MAX_SOFT_TOKENS
636
637
dummy_prompt = ("\t" + processor.image_token) * len(frames)