[Bugfix][Gemma 4] Don't read fft_length when profiling unified audio (#56721)
Signed-off-by: Abhinav Verma <verma.abhinav275@gmail.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Abhinav Verma committed
Sep 15, 2026 at 20:27 UTC
1e47ec00d2e6ad911969508068b72f7d1aae1968
2 files changed
+67
-2
tests/models/multimodal/processing/test_gemma4_unified.py
+43
-1
@@ -6,9 +6,13 @@ from collections.abc import Mapping
6
import pytest
7
import torch
8
from PIL import Image as PILImage
9
+from transformers.models.gemma4 import Gemma4AudioFeatureExtractor
10
11
from vllm.exceptions import VLLMValidationError
11
-from vllm.model_executor.models.gemma4_mm import Gemma4ImagePixelInputs
12
+from vllm.model_executor.models.gemma4_mm import (
13
+ Gemma4ImagePixelInputs,
14
+ _dummy_audio_num_samples,
15
+)
16
from vllm.multimodal import MULTIMODAL_REGISTRY
17
from vllm.multimodal.inputs import MultiModalFieldConfig
18
@@ -204,3 +208,41 @@ def test_limit_mm_per_prompt(
208
mm_items=processor.info.parse_mm_data(mm_data),
209
hf_processor_mm_kwargs={},
210
)
211
+
212
+
213
+def test_dummy_audio_length_falls_back_for_the_unified_extractor():
214
+ """The unified audio feature extractor does not define ``fft_length``.
215
+
216
+ ``Gemma4DummyInputsBuilder.get_dummy_mm_data`` read that attribute
217
+ unconditionally, so memory profiling raised ``AttributeError`` for every
218
+ unified Gemma 4 deployment in which audio is the modality with the largest
219
+ per-item token count -- which is the case as soon as ``audio_seq_length``
220
+ is raised above the video budget.
221
+ """
222
+
223
+ class _UnifiedFeatureExtractor:
224
+ sampling_rate = 16000
225
+
226
+ class _UnifiedProcessor:
227
+ feature_extractor = _UnifiedFeatureExtractor()
228
+ audio_seq_length = 3000
229
+ audio_ms_per_token = 40
230
+
231
+ # 3000 tokens * 40 ms = 120 s at 16 kHz
232
+ assert _dummy_audio_num_samples(_UnifiedProcessor()) == 120 * 16000
233
+
234
+
235
+def test_dummy_audio_length_still_uses_fft_length_for_the_tower_extractor():
236
+ """The mel/tower extractor keeps its existing behaviour."""
237
+
238
+ class _TowerFeatureExtractor(Gemma4AudioFeatureExtractor):
239
+ def __init__(self): # noqa: D107 - bypass the real feature-extractor init
240
+ self.fft_length = 512
241
+ self.sampling_rate = 16000
242
+
243
+ class _TowerProcessor:
244
+ feature_extractor = _TowerFeatureExtractor()
245
+ audio_seq_length = 750
246
+ audio_ms_per_token = 40
247
+
248
+ assert _dummy_audio_num_samples(_TowerProcessor()) == 512
vllm/model_executor/models/gemma4_mm.py
+24
-1
@@ -24,6 +24,7 @@ from PIL import Image as PILImage
24
from torch import nn
25
from transformers import AutoModel, BatchFeature
26
from transformers.models.gemma4 import (
27
+ Gemma4AudioFeatureExtractor,
28
Gemma4Config,
29
Gemma4Processor,
30
Gemma4VisionConfig,
@@ -456,6 +457,28 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
457
# ---------------------------------------------------------------------------
458
459
460
+def _dummy_audio_num_samples(processor: Any) -> int:
461
+ """Length in samples of a worst-case dummy audio item.
462
+
463
+ ``fft_length`` is set in ``Gemma4AudioFeatureExtractor.__init__`` and is not
464
+ defined by ``Gemma4UnifiedAudioFeatureExtractor``, which the encoder-free
465
+ Gemma 4 variant uses. Reading it unconditionally therefore raises
466
+ ``AttributeError`` during memory profiling on that variant, which is reached
467
+ whenever audio is the modality with the largest per-item token count.
468
+
469
+ For any other extractor, fall back to the audio budget the processor itself
470
+ declares -- the same pair of values used to warn about over-long audio
471
+ elsewhere in this file.
472
+ """
473
+ feature_extractor = processor.feature_extractor
474
+ if isinstance(feature_extractor, Gemma4AudioFeatureExtractor):
475
+ return feature_extractor.fft_length
476
+
477
+ sampling_rate = feature_extractor.sampling_rate
478
+ max_duration_s = processor.audio_seq_length * processor.audio_ms_per_token / 1000.0
479
+ return int(max_duration_s * sampling_rate)
480
+
481
+
482
class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
483
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
484
num_images = mm_counts.get("image", 0)
@@ -508,7 +531,7 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
531
}
532
533
if num_audios > 0:
511
- audio_len = processor.feature_extractor.fft_length
534
+ audio_len = _dummy_audio_num_samples(processor)
535
data["audio"] = self._get_dummy_audios(
536
length=audio_len,
537
num_audios=num_audios,