@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a248b45d05
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 2, 01:00 UTC

[Model] Add Gemma4 Unified (encoder-free) support (#44429)

Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>

Luciano Martins committed Jun 3, 2026 at 16:01 UTC a248b45d0548d2db110d45799340bf525cbce0e8
14 files changed +791 -31
docs/features/speculative_decoding/mtp.md
+5 -4
@@ -24,10 +24,11 @@ vllm serve google/gemma-4-E2B-it \
24 --speculative-config '{"method":"mtp","model":"gg-hf-am/gemma-4-E2B-it-assistant","num_speculative_tokens":1}'
25 ```
26
27 -The E2B, E4B, 26B-A4B, and 31B Gemma 4 IT assistant checkpoints are supported
28 -when their configuration uses `model_type: gemma4_assistant`. vLLM maps those
29 -checkpoints to `Gemma4MTPModel` internally and wires the assistant layers to
30 -share KV cache with the target model.
27 +The E2B, E4B, 12B, 26B-A4B, and 31B Gemma 4 IT assistant checkpoints are supported.
28 +Tower-based variants use `model_type: gemma4_assistant` and the encoder-free
29 +Gemma 4 Unified variant (12B) uses `model_type: gemma4_unified_assistant`.
30 +vLLM maps both to `Gemma4MTPModel` internally and wires the assistant layers
31 +to share KV cache with the target model.
32
33 If an older vLLM release logs `SpeculativeConfig(method='draft_model', ...)`
34 for a Gemma 4 assistant checkpoint, that release is treating the assistant as a
docs/models/supported_models.md
+8 -1
@@ -562,6 +562,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
562 | `Gemma3ForConditionalGeneration` | Gemma 3 | T + I<sup>E+</sup> | `google/gemma-3-4b-it`, `google/gemma-3-27b-it`, etc. | ✅︎ | ✅︎ |
563 | `Gemma3nForConditionalGeneration` | Gemma 3n | T + I + A | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | |
564 | `Gemma4ForConditionalGeneration` | Gemma 4 | T + I<sup>+</sup> + V + A<sup>*</sup> | `google/gemma-4-E2B-it`, etc. | | ✅︎ |
565 +| `Gemma4UnifiedForConditionalGeneration` | Gemma 4 Unified | T + I<sup>+</sup> + V + A | `google/gemma-4-12B-it`, etc. | | ✅︎ |
566 | `GLM4VForCausalLM`<sup>^</sup> | GLM-4V | T + I | `zai-org/glm-4v-9b`, `zai-org/cogagent-9b-20241220`, etc. | ✅︎ | ✅︎ |
567 | `Glm4vForConditionalGeneration` | GLM-4.1V-Thinking | T + I<sup>E+</sup> + V<sup>E+</sup> | `zai-org/GLM-4.1V-9B-Thinking`, etc. | ✅︎ | ✅︎ |
568 | `Glm4vMoeForConditionalGeneration` | GLM-4.5V | T + I<sup>E+</sup> + V<sup>E+</sup> | `zai-org/GLM-4.5V`, etc. | ✅︎ | ✅︎ |
@@ -664,10 +665,16 @@ Some models are supported only via the [Transformers modeling backend](#transfor
665 For `Gemma4ForConditionalGeneration`:
666 - audio input is only supported by the `gemma-4-E2B` and `gemma-4-E4B` variants.
667 - The model does not ingest videos directly. However, vLLM’s Gemma 4 implementation supports video inputs by handling video processing internally. Users can send videos directly in the message structure to vLLM, where they are converted into text and image frames before being passed to the model.
667 - - Gemma 4 assistant checkpoints for speculative decoding use vLLM's Gemma
668 + - Gemma 4 assistant checkpoints for speculative decoding use vLLM’s Gemma
669 4 MTP path, not generic draft-model speculative decoding. See the
670 [Gemma 4 assistant model MTP example](../features/speculative_decoding/mtp.md#gemma-4-assistant-models).
671
672 +!!! note
673 + For `Gemma4UnifiedForConditionalGeneration`:
674 + - This is the encoder-free Gemma 4 variant (e.g. `gemma-4-12B-it`). Unlike the tower-based `Gemma4ForConditionalGeneration`, it has **no SigLIP vision encoder** and **no audio encoder**. Raw pixel patches are projected directly into LM space via a Dense+LayerNorm pipeline with factorized positional embeddings, and raw audio waveform frames are projected directly through a multimodal embedder.
675 + - All modalities (image, video, audio) are supported.
676 + - Gemma 4 Unified assistant checkpoints (`model_type: gemma4_unified_assistant`) use the same MTP path as the tower-based variant. See the [Gemma 4 assistant model MTP example](../features/speculative_decoding/mtp.md#gemma-4-assistant-models).
677 +
678 !!! note
679 For `InternVLChatModel`, only InternVL2.5 with Qwen2.5 text backbone (`OpenGVLab/InternVL2.5-1B` etc.), InternVL3 and InternVL3.5 have video inputs support currently.
680
tests/models/multimodal/processing/test_gemma4_unified.py new
+205
@@ -0,0 +1,205 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from collections.abc import Mapping
5 +
6 +import pytest
7 +import torch
8 +from PIL import Image as PILImage
9 +
10 +from vllm.model_executor.models.gemma4_mm import Gemma4ImagePixelInputs
11 +from vllm.multimodal import MULTIMODAL_REGISTRY
12 +from vllm.multimodal.inputs import MultiModalFieldConfig
13 +
14 +from ....conftest import ImageTestAssets
15 +from ...utils import build_model_context
16 +
17 +# The Unified model ID for testing purposes
18 +GEMMA4_UNIFIED_MODEL_ID = "google/gemma-4-12B-it"
19 +
20 +
21 +def test_gemma4_unified_image_schema_accepts_variable_patch_counts():
22 + Gemma4ImagePixelInputs(
23 + pixel_values=[
24 + torch.randn(10080, 768),
25 + torch.randn(2520, 768),
26 + ],
27 + pixel_position_ids=[
28 + torch.zeros(10080, 2, dtype=torch.long),
29 + torch.zeros(2520, 2, dtype=torch.long),
30 + ],
31 + )
32 +
33 +
34 +def test_gemma4_unified_image_batching_keeps_variable_patch_counts_unstacked():
35 + field = MultiModalFieldConfig.batched("image").field
36 + elems = field.build_elems(
37 + "image",
38 + "pixel_values",
39 + [torch.randn(10080, 768), torch.randn(2520, 768)],
40 + )
41 +
42 + reduced = field.reduce_data(list(elems))
43 +
44 + assert isinstance(reduced, list)
45 + assert [tensor.shape for tensor in reduced] == [
46 + torch.Size([10080, 768]),
47 + torch.Size([2520, 768]),
48 + ]
49 +
50 +
51 +@pytest.mark.parametrize(
52 + "image_width,image_height,max_soft_tokens",
53 + [
54 + (900, 3, 280),
55 + (3, 900, 280),
56 + (900, 3, 70),
57 + (4000, 2, 1120),
58 + ],
59 +)
60 +@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
61 +def test_compute_num_soft_tokens_does_not_exceed_max_soft_tokens(
62 + model_id: str,
63 + image_width: int,
64 + image_height: int,
65 + max_soft_tokens: int,
66 +):
67 + """Verify ``_compute_num_soft_tokens`` caps output at ``max_soft_tokens``."""
68 + ctx = build_model_context(
69 + model_id,
70 + mm_processor_kwargs={"do_pan_and_scan": True},
71 + limit_mm_per_prompt={"image": 1},
72 + )
73 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
74 +
75 + num_soft_tokens = processor.info._compute_num_soft_tokens(
76 + image_width=image_width,
77 + image_height=image_height,
78 + max_soft_tokens=max_soft_tokens,
79 + )
80 +
81 + assert num_soft_tokens <= max_soft_tokens, (
82 + f"_compute_num_soft_tokens returned {num_soft_tokens} for "
83 + f"image_width={image_width}, image_height={image_height}, "
84 + f"max_soft_tokens={max_soft_tokens} — exceeds the cap."
85 + )
86 +
87 +
88 +@pytest.mark.parametrize(
89 + ("mm_processor_kwargs", "expected_image_tokens"),
90 + [
91 + ({}, 280),
92 + ({"max_soft_tokens": 70}, 70),
93 + ({"max_soft_tokens": 280}, 280),
94 + ({"max_soft_tokens": 1120}, 1120),
95 + ({"images_kwargs": {"max_soft_tokens": 560}}, 560),
96 + ({"images_kwargs": None}, 280),
97 + ({"images_kwargs": "not-a-dict"}, 280),
98 + ],
99 +)
100 +@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
101 +def test_get_mm_max_tokens_per_item_respects_configured_max_soft_tokens(
102 + model_id: str,
103 + mm_processor_kwargs: dict[str, object],
104 + expected_image_tokens: int,
105 +):
106 + ctx = build_model_context(
107 + model_id,
108 + mm_processor_kwargs=mm_processor_kwargs,
109 + limit_mm_per_prompt={"image": 1, "video": 1},
110 + )
111 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
112 +
113 + tokens = processor.info.get_mm_max_tokens_per_item(
114 + seq_len=ctx.model_config.max_model_len,
115 + mm_counts={"image": 1, "video": 1},
116 + )
117 +
118 + assert tokens is not None
119 + assert tokens["image"] == expected_image_tokens
120 + assert tokens["video"] == 32 * (70 + 2 + 6)
121 +
122 +
123 +@pytest.mark.parametrize(
124 + ("limit_mm_per_prompt", "expected_video_tokens"),
125 + [
126 + ({"video": 1}, 32 * (70 + 2 + 6)),
127 + ({"video": {"count": 1}}, 32 * (70 + 2 + 6)),
128 + ({"video": {"count": 1, "num_frames": 1}}, 1 * (70 + 2 + 6)),
129 + ({"video": {"count": 1, "num_frames": 8}}, 8 * (70 + 2 + 6)),
130 + ({"video": {"count": 1, "num_frames": 32}}, 32 * (70 + 2 + 6)),
131 + ({"video": {"count": 1, "num_frames": 40}}, 32 * (70 + 2 + 6)),
132 + ],
133 +)
134 +@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
135 +def test_get_mm_max_tokens_per_item_respects_configured_video_num_frames(
136 + model_id: str,
137 + limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]],
138 + expected_video_tokens: int,
139 +):
140 + ctx = build_model_context(
141 + model_id,
142 + limit_mm_per_prompt=limit_mm_per_prompt,
143 + )
144 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
145 +
146 + tokens = processor.info.get_mm_max_tokens_per_item(
147 + seq_len=ctx.model_config.max_model_len,
148 + mm_counts={"video": 1},
149 + )
150 +
151 + assert tokens is not None
152 + assert tokens["image"] == 280
153 + assert tokens["video"] == expected_video_tokens
154 +
155 +
156 +@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
157 +def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
158 + ctx = build_model_context(
159 + model_id,
160 + mm_processor_kwargs={"images_kwargs": {"max_soft_tokens": 560}},
161 + limit_mm_per_prompt={"image": 1},
162 + )
163 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
164 + image = PILImage.new("RGB", (1000, 1000), color="white")
165 + image_size = image.size
166 + mm_items = processor.info.parse_mm_data({"image": image})
167 +
168 + prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
169 + replacement = prompt_update.resolve(0).content.full
170 + expected = processor.info.get_image_repl(
171 + image_width=image_size[0],
172 + image_height=image_size[1],
173 + processor=processor.info.get_hf_processor(),
174 + max_soft_tokens=560,
175 + ).full
176 +
177 + assert replacement == expected
178 +
179 +
180 +@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
181 +def test_limit_mm_per_prompt(
182 + image_assets: ImageTestAssets,
183 + model_id: str,
184 +):
185 + """Test that limit_mm_per_prompt restricts multiple images correctly."""
186 + ctx = build_model_context(
187 + model_id,
188 + mm_processor_kwargs={},
189 + limit_mm_per_prompt={"image": 1},
190 + )
191 + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
192 +
193 + prompt = "<image><image>"
194 + images = [asset.pil_image for asset in image_assets][:2]
195 + if len(images) < 2:
196 + images = [images[0], images[0]]
197 +
198 + mm_data = {"image": images}
199 +
200 + with pytest.raises(ValueError, match="At most 1 image"):
201 + processor(
202 + prompt,
203 + mm_items=processor.info.parse_mm_data(mm_data),
204 + hf_processor_mm_kwargs={},
205 + )
tests/models/registry.py
+7
@@ -917,6 +917,13 @@ _MULTIMODAL_EXAMPLE_MODELS = {
917 "google/gemma-4-E2B-it",
918 min_transformers_version="5.5.0",
919 ),
920 + # TODO: update min_transformers_version when Gemma4 Unified lands in
921 + # a stable transformers release.
922 + "Gemma4UnifiedForConditionalGeneration": _HfExamplesInfo(
923 + "google/gemma-4-12B-it",
924 + min_transformers_version="5.8.0",
925 + is_available_online=False,
926 + ),
927 "Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"),
928 "GlmAsrForConditionalGeneration": _HfExamplesInfo(
929 "zai-org/GLM-ASR-Nano-2512",
vllm/config/speculative.py
+1 -1
@@ -509,7 +509,7 @@ class SpeculativeConfig:
509 {"n_predict": n_predict, "architectures": ["HYV3MTPModel"]}
510 )
511
512 - if hf_config.model_type == "gemma4_assistant":
512 + if hf_config.model_type in ("gemma4_assistant", "gemma4_unified_assistant"):
513 hf_config.model_type = "gemma4_mtp"
514 text_config = getattr(hf_config, "text_config", hf_config)
515 # The assistant runs all decoder layers in a single forward
vllm/model_executor/models/config.py
+1
@@ -597,6 +597,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
597 "Gemma3TextModel": Gemma3TextModelConfig,
598 "Gemma4ForCausalLM": Gemma4Config,
599 "Gemma4ForConditionalGeneration": Gemma4Config,
600 + "Gemma4UnifiedForConditionalGeneration": Gemma4Config,
601 "GptOssForCausalLM": GptOssForCausalLMConfig,
602 "GteModel": SnowflakeGteNewModelConfig,
603 "GteNewForSequenceClassification": GteNewModelConfig,
vllm/model_executor/models/gemma4.py
+6 -3
@@ -1051,11 +1051,14 @@ class Gemma4Model(nn.Module, EagleModelMixin):
1051 # Final norm: output = norm(x) * weight
1052 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
1053
1054 - # Embedding scale = sqrt(hidden_size)
1055 - # Downcast to model dtype (bfloat16 etc.) for numerical parity
1054 + # Embedding scale = sqrt(hidden_size), cast to model dtype to avoid
1055 + # mixed-precision drift from bf16 * fp32 across deep stacks.
1056 self.register_buffer(
1057 "normalizer",
1058 - torch.tensor(config.hidden_size**0.5),
1058 + torch.tensor(
1059 + config.hidden_size**0.5,
1060 + dtype=self.embed_tokens.weight.dtype,
1061 + ),
1062 persistent=False,
1063 )
1064
vllm/model_executor/models/gemma4_mm.py
+64 -19
@@ -121,7 +121,7 @@ class Gemma4ImagePixelInputs(TensorSchema):
121 - np: Number of patches (max_patches = max_soft_tokens * pooling_kernel_size²)
122 - pp: Patch pixels (patch_size² * 3)
123
124 - The HF Gemma4ImageProcessor outputs pixel_values as
124 + The Gemma4 image processor outputs pixel_values as
125 (batch, max_patches, patch_pixels) — already patchified with
126 zero-padding for patches beyond the real image content.
127 pixel_position_ids provides (x, y) coordinates per patch,
@@ -341,6 +341,29 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
341 )
342 return PromptUpdateDetails.select_token_id(token_ids, processor.image_token_id)
343
344 + @staticmethod
345 + def _compute_audio_num_tokens(
346 + num_samples: int, sampling_rate: int, audio_seq_length: int
347 + ) -> int:
348 + """Replicate the audio encoder's sequence-length arithmetic.
349 +
350 + Mirrors: mel framing (_unfold in Gemma4AudioFeatureExtractor)
351 + followed by two Conv2d subsampling layers (kernel=3, stride=2,
352 + semicausal padding top=1, bottom=1), capped at audio_seq_length.
353 + """
354 + frame_length = int(round(sampling_rate * 20.0 / 1000.0))
355 + hop_length = int(round(sampling_rate * 10.0 / 1000.0))
356 + frame_size_for_unfold = frame_length + 1
357 + pad_left = frame_length // 2
358 + padded_samples = num_samples + pad_left
359 + num_mel_frames = (padded_samples - frame_size_for_unfold) // hop_length + 1
360 + if num_mel_frames <= 0:
361 + return 0
362 + t = num_mel_frames
363 + for _ in range(2):
364 + t = (t + 2 - 3) // 2 + 1
365 + return min(t, audio_seq_length)
366 +
367 def get_audio_repl(
368 self,
369 *,
@@ -350,20 +373,21 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
373 """Return the dynamic audio token sequence for this audio.
374
375 Computes the number of soft tokens from the audio waveform
353 - length using ``ceil(duration_ms / audio_ms_per_token)``.
376 + length by replicating the audio encoder's sequence-length
377 + arithmetic (mel framing + two Conv2d subsampling layers).
378 """
379 if processor is None:
380 processor = self.get_hf_processor()
381
382 sampling_rate = processor.feature_extractor.sampling_rate
359 - num_tokens = processor._compute_audio_num_tokens(
360 - torch.zeros(audio_len), sampling_rate
383 + num_tokens = self._compute_audio_num_tokens(
384 + audio_len, sampling_rate, processor.audio_seq_length
385 )
386 config = self.get_hf_config()
387 token_ids = (
388 [config.boa_token_id]
389 + [processor.audio_token_id] * num_tokens
366 - + [config.eoa_token_id]
390 + + [getattr(config, "eoa_token_id", config.eoa_token_index)]
391 )
392 return PromptUpdateDetails.select_token_id(token_ids, processor.audio_token_id)
393
@@ -988,18 +1012,35 @@ class Gemma4ForConditionalGeneration(
1012 self.quant_config = quant_config
1013 self.multimodal_config = multimodal_config
1014
1015 + # Only quantize towers when the quant method supports their
1016 + # dimensions. BNB/torchao handle arbitrary sizes; other methods
1017 + # (Marlin, FP8, …) require dimensions divisible by 64, which
1018 + # the vision tower (intermediate_size=4304) does not satisfy.
1019 + if quant_config and quant_config.get_name() in [
1020 + "bitsandbytes",
1021 + "torchao",
1022 + ]:
1023 + tower_quant = quant_config
1024 + else:
1025 + vision_cfg = config.vision_config
1026 + quantizable = (
1027 + vision_cfg.hidden_size % 64 == 0
1028 + and vision_cfg.intermediate_size % 64 == 0
1029 + )
1030 + tower_quant = quant_config if quantizable else None
1031 +
1032 # ---- Vision tower (shared by image and video) ----
1033 with self._mark_tower_model(vllm_config, {"image", "video"}):
1034 self.vision_tower = AutoModel.from_config(config=config.vision_config)
1035 self.embed_vision = Gemma4MultimodalEmbedder(
1036 config.vision_config,
1037 config.text_config,
997 - quant_config=quant_config,
1038 + quant_config=tower_quant,
1039 prefix=maybe_prefix(prefix, "embed_vision"),
1040 )
1041 recursive_replace_linear(
1042 self.vision_tower,
1002 - quant_config,
1043 + tower_quant,
1044 prefix=maybe_prefix(prefix, "vision_tower"),
1045 )
1046
@@ -1015,12 +1056,12 @@ class Gemma4ForConditionalGeneration(
1056 self.embed_audio = Gemma4MultimodalEmbedder(
1057 config.audio_config,
1058 config.text_config,
1018 - quant_config=quant_config,
1059 + quant_config=tower_quant,
1060 prefix=maybe_prefix(prefix, "embed_audio"),
1061 )
1062 recursive_replace_linear(
1063 self.audio_tower,
1023 - quant_config,
1064 + tower_quant,
1065 prefix=maybe_prefix(prefix, "audio_tower"),
1066 )
1067 else:
@@ -1039,13 +1080,13 @@ class Gemma4ForConditionalGeneration(
1080 # Pre-allocate PLE buffer for CUDA graph compatibility.
1081 # Some variants have hidden_size_per_layer_input=None (no PLE).
1082 ple_dim = config.text_config.hidden_size_per_layer_input
1042 - if ple_dim is not None:
1083 + if ple_dim is not None and ple_dim > 0:
1084 self.per_layer_embeddings = torch.zeros(
1085 vllm_config.scheduler_config.max_num_batched_tokens,
1086 config.text_config.num_hidden_layers,
1087 ple_dim,
1047 - device=(self.language_model.model.embed_tokens.weight.device),
1048 - dtype=(self.language_model.model.embed_tokens.weight.dtype),
1088 + device=self.language_model.model.embed_tokens.weight.device,
1089 + dtype=self.language_model.model.embed_tokens.weight.dtype,
1090 )
1091 else:
1092 self.per_layer_embeddings = None
@@ -1076,6 +1117,9 @@ class Gemma4ForConditionalGeneration(
1117 self.num_shared_experts = self.language_model.num_shared_experts
1118 self.num_redundant_experts = self.language_model.num_redundant_experts
1119
1120 + gen_cfg = vllm_config.model_config.try_get_generation_config()
1121 + self._suppress_token_ids = gen_cfg.get("suppress_tokens") if gen_cfg else None
1122 +
1123 # ------------------------------------------------------------------ #
1124 # Input parsing
1125 # ------------------------------------------------------------------ #
@@ -1424,8 +1468,7 @@ class Gemma4ForConditionalGeneration(
1468 input_features = audio_input["input_features_padded"].squeeze(1)
1469 input_features_mask = audio_input["input_features_mask"].squeeze(1)
1470
1427 - # Run audio tower — mask uses standard HF convention
1428 - # (True=valid, False=padding).
1471 + # Run audio tower — mask convention: True=valid, False=padding.
1472 audio_outputs = self.audio_tower(input_features, input_features_mask)
1473 if isinstance(audio_outputs, tuple):
1474 audio_encodings, audio_mask = audio_outputs
@@ -1436,8 +1479,8 @@ class Gemma4ForConditionalGeneration(
1479 # Project into LM embedding space.
1480 audio_features = self.embed_audio(inputs_embeds=audio_encodings)
1481
1439 - # Strip padding per-batch element: only keep real (non-padding)
1440 - # tokens. audio_mask is True for valid positions (HF convention).
1482 + # Strip padding per-batch element: only keep valid (non-padding)
1483 + # tokens.
1484 per_audio = []
1485 for enc, mask in zip(audio_features, audio_mask, strict=True):
1486 per_audio.append(enc[mask]) # [num_real, hidden_size]
@@ -1559,7 +1602,10 @@ class Gemma4ForConditionalGeneration(
1602 self,
1603 hidden_states: torch.Tensor,
1604 ) -> torch.Tensor | None:
1562 - return self.language_model.compute_logits(hidden_states)
1605 + logits = self.language_model.compute_logits(hidden_states)
1606 + if logits is not None and self._suppress_token_ids:
1607 + logits[:, self._suppress_token_ids] = -float("inf")
1608 + return logits
1609
1610 # ------------------------------------------------------------------ #
1611 # Bidirectional attention helpers
@@ -1617,8 +1663,7 @@ class Gemma4ForConditionalGeneration(
1663 "embed_vision.embedding.",
1664 "embed_audio.embedding.",
1665 ]
1620 - # Models without audio tower should skip
1621 - # audio weights entirely.
1666 + # Models without audio tower should skip audio weights entirely.
1667 if self.audio_tower is None:
1668 ignore_prefixes.extend(
1669 [
vllm/model_executor/models/gemma4_mtp.py
+19 -3
@@ -279,11 +279,19 @@ class Gemma4MTPDecoderLayer(nn.Module):
279 else config.head_dim
280 )
281
282 + use_k_eq_v = is_full_attention and getattr(config, "attention_k_eq_v", False)
283 + if use_k_eq_v:
284 + num_kv_heads = getattr(
285 + config, "num_global_key_value_heads", config.num_key_value_heads
286 + )
287 + else:
288 + num_kv_heads = config.num_key_value_heads
289 +
290 self.self_attn = Gemma4MTPAttention(
291 config=config,
292 hidden_size=self.hidden_size,
293 num_heads=config.num_attention_heads,
286 - num_kv_heads=config.num_key_value_heads,
294 + num_kv_heads=num_kv_heads,
295 head_dim=head_dim,
296 max_position_embeddings=config.max_position_embeddings,
297 cache_config=cache_config,
@@ -545,6 +553,10 @@ class Gemma4MTP(nn.Module):
553 else:
554 self.masked_embedding = None
555
556 + draft_cfg = vllm_config.speculative_config.draft_model_config
557 + gen_cfg = draft_cfg.try_get_generation_config()
558 + self._suppress_token_ids = gen_cfg.get("suppress_tokens") if gen_cfg else None
559 +
560 def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
561 return self.model.embed_input_ids(input_ids)
562
@@ -589,11 +601,15 @@ class Gemma4MTP(nn.Module):
601 spec_step_idx: int = 0,
602 ) -> torch.Tensor | None:
603 if self.masked_embedding is not None:
592 - return self.masked_embedding(
604 + logits = self.masked_embedding(
605 hidden_states,
606 self._get_full_lm_head_weight(),
607 )
596 - return self.logits_processor(self.lm_head, hidden_states)
608 + else:
609 + logits = self.logits_processor(self.lm_head, hidden_states)
610 + if logits is not None and self._suppress_token_ids:
611 + logits[:, self._suppress_token_ids] = -float("inf")
612 + return logits
613
614 def get_top_tokens(
615 self,
vllm/model_executor/models/gemma4_unified.py new
+466
@@ -0,0 +1,466 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Gemma 4 Unified multimodal model (encoder-free image + audio + video).
4 +
5 +The Unified Gemma4 variant has no SigLIP vision tower and no audio tower.
6 +Raw pixel patches are projected directly to LM space via a Dense+LayerNorm
7 +pipeline with factorized 2D positional embeddings (Gemma4UnifiedVisionEmbedder),
8 +then routed through the same Gemma4MultimodalEmbedder used by the tower-based
9 +variant. Audio inputs are raw waveform frames projected directly through the
10 +multimodal embedder.
11 +
12 +This module subclasses Gemma4ForConditionalGeneration from gemma4_mm rather
13 +than reimplementing it from scratch. Only the multimodal pipeline differs;
14 +the language model, MTP integration, bidirectional attention helpers,
15 +embedding/forward path, and LoRA support are all inherited unchanged.
16 +"""
17 +
18 +import math
19 +from collections.abc import Iterable, Mapping
20 +
21 +import torch
22 +from torch import nn
23 +from transformers.models.gemma4_unified.configuration_gemma4_unified import (
24 + Gemma4UnifiedConfig,
25 +)
26 +from transformers.models.gemma4_unified.processing_gemma4_unified import (
27 + Gemma4UnifiedProcessor,
28 +)
29 +
30 +from vllm.config import VllmConfig
31 +from vllm.config.multimodal import VideoDummyOptions
32 +from vllm.model_executor.layers.linear import ColumnParallelLinear
33 +from vllm.model_executor.models.gemma4 import Gemma4ForCausalLM
34 +from vllm.model_executor.models.gemma4_mm import (
35 + _SUPPORTED_SOFT_TOKENS,
36 + _VIDEO_MAX_FRAMES,
37 + _VIDEO_MAX_SOFT_TOKENS,
38 + Gemma4AudioInputs,
39 + Gemma4DummyInputsBuilder,
40 + Gemma4ForConditionalGeneration,
41 + Gemma4ImageInputs,
42 + Gemma4ImagePixelInputs,
43 + Gemma4MultimodalEmbedder,
44 + Gemma4MultiModalProcessor,
45 + Gemma4ProcessingInfo,
46 + _get_max_soft_tokens,
47 +)
48 +from vllm.model_executor.models.module_mapping import MultiModelKeys
49 +from vllm.multimodal import MULTIMODAL_REGISTRY
50 +
51 +from .utils import (
52 + AutoWeightsLoader,
53 + WeightsMapper,
54 + init_vllm_registered_model,
55 + maybe_prefix,
56 +)
57 +
58 +# Re-export so tests/code targeting the unified variant can import from here
59 +# rather than reaching into gemma4_mm.
60 +__all__ = [
61 + "Gemma4ImagePixelInputs",
62 + "Gemma4UnifiedVisionEmbedder",
63 + "Gemma4UnifiedProcessingInfo",
64 + "Gemma4UnifiedForConditionalGeneration",
65 +]
66 +
67 +
68 +# ---------------------------------------------------------------------------
69 +# Encoder-free vision embedder
70 +# ---------------------------------------------------------------------------
71 +
72 +
73 +class Gemma4UnifiedVisionEmbedder(nn.Module):
74 + """Encoder-free vision embedder for Gemma4 Unified variants.
75 +
76 + Projects raw pixel patches to LM space via dense projection and
77 + factorized 2D positional embeddings. Replaces the SigLIP vision
78 + tower used by the tower-based Gemma4 variant.
79 +
80 + Pipeline: raw patches → LN₁ → Dense → LN₂ → +factorized_posemb → LN₃.
81 + """
82 +
83 + def __init__(self, config, quant_config=None):
84 + super().__init__()
85 + patch_dim = config.model_patch_size**2 * 3
86 + mm_embed_dim = config.mm_embed_dim
87 +
88 + self.patch_ln1 = nn.LayerNorm(patch_dim)
89 + self.patch_dense = ColumnParallelLinear(
90 + patch_dim,
91 + mm_embed_dim,
92 + bias=True,
93 + quant_config=quant_config,
94 + gather_output=True,
95 + )
96 + self.patch_ln2 = nn.LayerNorm(mm_embed_dim)
97 +
98 + self.pos_embedding = nn.Parameter(
99 + torch.zeros(config.mm_posemb_size, 2, mm_embed_dim)
100 + )
101 + self.pos_norm = nn.LayerNorm(mm_embed_dim)
102 +
103 + def _factorized_posemb(self, positions_xy: torch.Tensor) -> torch.Tensor:
104 + clamped_pos = positions_xy.clamp(min=0).long()
105 + valid_mask = positions_xy != -1
106 +
107 + pos_embs = torch.zeros(
108 + *positions_xy.shape[:-1],
109 + self.pos_embedding.shape[-1],
110 + device=positions_xy.device,
111 + dtype=self.pos_embedding.dtype,
112 + )
113 + for i in range(2):
114 + axis_pe = self.pos_embedding[:, i, :][clamped_pos[..., i]]
115 + mask = valid_mask[..., i].unsqueeze(-1).to(axis_pe.dtype)
116 + pos_embs = pos_embs + (axis_pe * mask)
117 + return pos_embs
118 +
119 + def forward(
120 + self,
121 + pixel_values: torch.Tensor,
122 + pixel_position_ids: torch.Tensor,
123 + ) -> torch.Tensor:
124 + hidden_states = self.patch_ln1(pixel_values.to(self.pos_embedding.dtype))
125 + hidden_states, _ = self.patch_dense(hidden_states)
126 + hidden_states = self.patch_ln2(hidden_states)
127 +
128 + pos_embs = self._factorized_posemb(pixel_position_ids)
129 + hidden_states = hidden_states + pos_embs
130 + hidden_states = self.pos_norm(hidden_states)
131 + return hidden_states
132 +
133 +
134 +# ---------------------------------------------------------------------------
135 +# Processing info
136 +# ---------------------------------------------------------------------------
137 +
138 +
139 +class Gemma4UnifiedProcessingInfo(Gemma4ProcessingInfo):
140 + """ProcessingInfo for the Gemma4 Unified variant.
141 +
142 + Two field-name differences from the tower-based parent:
143 + * config → ``Gemma4UnifiedConfig`` (not ``Gemma4Config``)
144 + * vision_config.``num_soft_tokens`` (not ``default_output_length``)
145 +
146 + Everything else (token sequencing, audio limits, video frame budget,
147 + parser construction) is inherited unchanged.
148 + """
149 +
150 + def get_hf_config(self):
151 + return self.ctx.get_hf_config(Gemma4UnifiedConfig)
152 +
153 + def get_hf_processor(self, **kwargs: object) -> Gemma4UnifiedProcessor:
154 + return self.ctx.get_hf_processor(
155 + Gemma4UnifiedProcessor,
156 + **kwargs,
157 + )
158 +
159 + def get_mm_max_tokens_per_item(
160 + self, seq_len: int, mm_counts: Mapping[str, int]
161 + ) -> Mapping[str, int] | None:
162 + config = self.get_hf_config()
163 + # Unified field is `num_soft_tokens`. Tower-based parent uses
164 + # `default_output_length`, hence the override.
165 + tokens_per_image = config.vision_config.num_soft_tokens
166 + merged_kwargs = self.ctx.get_merged_mm_kwargs({})
167 + val, _ = _get_max_soft_tokens(merged_kwargs)
168 + if isinstance(val, int) and val in _SUPPORTED_SOFT_TOKENS:
169 + tokens_per_image = val
170 + tokens: dict[str, int] = {"image": tokens_per_image}
171 + if config.audio_config is not None:
172 + processor = self.get_hf_processor()
173 + tokens["audio"] = processor.audio_seq_length
174 + num_frames = _VIDEO_MAX_FRAMES
175 + mm_config = self.ctx.model_config.get_multimodal_config()
176 + video_opts = mm_config.limit_per_prompt.get("video")
177 + if (
178 + isinstance(video_opts, VideoDummyOptions)
179 + and video_opts.num_frames is not None
180 + ):
181 + num_frames = min(num_frames, video_opts.num_frames)
182 + tokens["video"] = num_frames * (_VIDEO_MAX_SOFT_TOKENS + 2 + 6)
183 + return tokens
184 +
185 + def _compute_num_soft_tokens(
186 + self,
187 + image_width: int,
188 + image_height: int,
189 + max_soft_tokens: int | None = None,
190 + ) -> int:
191 + vision_cfg = self.get_hf_config().vision_config
192 + patch_size = vision_cfg.patch_size
193 + pooling_kernel_size = vision_cfg.pooling_kernel_size
194 +
195 + if max_soft_tokens is None:
196 + max_soft_tokens = vision_cfg.num_soft_tokens
197 +
198 + unit = patch_size * pooling_kernel_size
199 + max_patches = max_soft_tokens * pooling_kernel_size**2
200 + num_patches_orig = (image_height / patch_size) * (image_width / patch_size)
201 + scale = math.sqrt(max_patches / num_patches_orig)
202 + target_h = max(unit, int(math.floor(image_height * scale / unit)) * unit)
203 + target_w = max(unit, int(math.floor(image_width * scale / unit)) * unit)
204 + num_patches = (target_h // patch_size) * (target_w // patch_size)
205 + num_soft_tokens = num_patches // (pooling_kernel_size**2)
206 + return min(num_soft_tokens, max_soft_tokens)
207 +
208 +
209 +# ---------------------------------------------------------------------------
210 +# Main model
211 +# ---------------------------------------------------------------------------
212 +
213 +
214 +@MULTIMODAL_REGISTRY.register_processor(
215 + Gemma4MultiModalProcessor,
216 + info=Gemma4UnifiedProcessingInfo,
217 + dummy_inputs=Gemma4DummyInputsBuilder,
218 +)
219 +class Gemma4UnifiedForConditionalGeneration(Gemma4ForConditionalGeneration):
220 + """Encoder-free Gemma4 (Unified) for conditional generation.
221 +
222 + Inherits multimodal embedding routing, PLE handling, bidirectional
223 + attention helpers, language-model forward, LoRA, and pipeline-parallel
224 + support from :class:`Gemma4ForConditionalGeneration`. Overrides only:
225 +
226 + * ``__init__`` — builds the encoder-free vision embedder instead of
227 + SigLIP/audio towers (LightOnOCR-style: ``nn.Module.__init__`` +
228 + full rebuild, no ``super().__init__()``).
229 + * ``hf_to_vllm_mapper`` — adds the ``model.vision_embedder.`` prefix.
230 + * ``_process_image_input`` / ``_process_video_input`` /
231 + ``_process_audio_input`` — encoder-free projection paths.
232 + * ``load_weights`` — ignore-prefix list excludes the absent towers.
233 + * ``get_mm_mapping`` — no tower entries.
234 + """
235 +
236 + hf_to_vllm_mapper = WeightsMapper(
237 + orig_to_new_prefix={
238 + "model.embed_audio.": "embed_audio.",
239 + "model.embed_vision.": "embed_vision.",
240 + "model.language_model.": "language_model.model.",
241 + "model.vision_embedder.": "vision_embedder.",
242 + "lm_head.": "language_model.lm_head.",
243 + "model": "language_model.model",
244 + }
245 + )
246 +
247 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
248 + # LightOnOCR-style rebuild: do NOT call super().__init__ — that
249 + # would build a SigLIP vision tower and an audio tower we don't
250 + # need. Initialize nn.Module directly and assemble the
251 + # encoder-free pipeline below.
252 + nn.Module.__init__(self)
253 + config = vllm_config.model_config.hf_config
254 + quant_config = vllm_config.quant_config
255 + multimodal_config = vllm_config.model_config.multimodal_config
256 + self.config = config
257 + self.quant_config = quant_config
258 + self.multimodal_config = multimodal_config
259 +
260 + # No towers — set to None so inherited load_weights / get_mm_mapping
261 + # and any tower-aware logic short-circuits.
262 + self.vision_tower = None
263 + self.audio_tower = None
264 +
265 + # ---- Encoder-free vision embedder ----
266 + self.vision_embedder = (
267 + Gemma4UnifiedVisionEmbedder(
268 + config.vision_config,
269 + quant_config=quant_config,
270 + )
271 + if config.vision_config is not None
272 + else None
273 + )
274 + self.embed_vision = (
275 + Gemma4MultimodalEmbedder(
276 + config.vision_config,
277 + config.text_config,
278 + )
279 + if config.vision_config is not None
280 + else None
281 + )
282 +
283 + # ---- Encoder-free audio embedder ----
284 + self.embed_audio = (
285 + Gemma4MultimodalEmbedder(
286 + config.audio_config,
287 + config.text_config,
288 + )
289 + if config.audio_config is not None
290 + else None
291 + )
292 +
293 + # ---- Language model (vLLM optimised) ----
294 + with self._mark_language_model(vllm_config):
295 + self.language_model: Gemma4ForCausalLM = init_vllm_registered_model(
296 + vllm_config=vllm_config,
297 + hf_config=config.text_config,
298 + prefix=maybe_prefix(prefix, "language_model"),
299 + architectures=["Gemma4ForCausalLM"],
300 + )
301 +
302 + # PLE is disabled for the unified variant (text config defaults
303 + # hidden_size_per_layer_input to 0). Skip the buffer.
304 + ple_dim = getattr(
305 + config.text_config,
306 + "hidden_size_per_layer_input",
307 + None,
308 + )
309 + if ple_dim is not None and ple_dim > 0:
310 + self.per_layer_embeddings = torch.zeros(
311 + vllm_config.scheduler_config.max_num_batched_tokens,
312 + config.text_config.num_hidden_layers,
313 + ple_dim,
314 + device=self.language_model.model.embed_tokens.weight.device,
315 + dtype=self.language_model.model.embed_tokens.weight.dtype,
316 + )
317 + else:
318 + self.per_layer_embeddings = None
319 +
320 + self.make_empty_intermediate_tensors = (
321 + self.language_model.make_empty_intermediate_tensors
322 + )
323 +
324 + # --- Precompute full-attention layer indices for bidi clearing ---
325 + self._full_attn_layer_idxs: frozenset[int] = frozenset()
326 + text_config = config.text_config
327 + if getattr(text_config, "use_bidirectional_attention", None) == "vision":
328 + layer_types = getattr(text_config, "layer_types", None)
329 + if layer_types:
330 + self._full_attn_layer_idxs = frozenset(
331 + i for i, lt in enumerate(layer_types) if lt != "sliding_attention"
332 + )
333 +
334 + # --- MixtureOfExperts delegation to language_model ---
335 + self.expert_weights = self.language_model.expert_weights
336 + self.moe_layers = self.language_model.moe_layers
337 + self.num_moe_layers = self.language_model.num_moe_layers
338 + self.num_logical_experts = self.language_model.num_logical_experts
339 + self.num_physical_experts = self.language_model.num_physical_experts
340 + self.num_local_physical_experts = self.language_model.num_local_physical_experts
341 + self.num_routed_experts = self.language_model.num_routed_experts
342 + self.num_expert_groups = self.language_model.num_expert_groups
343 + self.num_shared_experts = self.language_model.num_shared_experts
344 + self.num_redundant_experts = self.language_model.num_redundant_experts
345 +
346 + gen_cfg = vllm_config.model_config.try_get_generation_config()
347 + self._suppress_token_ids = gen_cfg.get("suppress_tokens") if gen_cfg else None
348 +
349 + # ------------------------------------------------------------------ #
350 + # Multimodal processing (encoder-free overrides)
351 + # ------------------------------------------------------------------ #
352 +
353 + def _process_image_input(
354 + self,
355 + image_input: Gemma4ImageInputs,
356 + ) -> list[torch.Tensor]:
357 + """Project raw image patches directly to LM space.
358 +
359 + No vision tower: each image's pre-patchified pixel values are
360 + embedded via Gemma4UnifiedVisionEmbedder, projected through
361 + Gemma4MultimodalEmbedder, and padding patches (pp == -1) are
362 + stripped per image.
363 + """
364 + pixel_values = image_input["pixel_values"]
365 + pixel_position_ids = image_input["pixel_position_ids"]
366 + target_dtype = self.embed_vision.embedding_projection.weight.dtype
367 +
368 + per_image_features: list[torch.Tensor] = []
369 + for pv, pp in zip(pixel_values, pixel_position_ids, strict=True):
370 + pv = pv.unsqueeze(0)
371 + pp = pp.unsqueeze(0)
372 + embedded = self.vision_embedder(pv, pp)
373 + projected = self.embed_vision(embedded.to(target_dtype))
374 + padding_mask = (pp.squeeze(0) == -1).all(dim=-1)
375 + valid_features = projected.squeeze(0)[~padding_mask]
376 + per_image_features.append(valid_features)
377 + return per_image_features
378 +
379 + def _process_video_input(
380 + self,
381 + video_input: dict[str, torch.Tensor],
382 + ) -> list[torch.Tensor]:
383 + """Project video frames to LM space, one frame at a time.
384 +
385 + Frames are split per video, each frame is embedded + projected,
386 + and per-frame valid embeddings are concatenated per video.
387 + """
388 + pixel_values = video_input["pixel_values_videos"]
389 + pixel_position_ids = video_input["pixel_position_ids_videos"]
390 + frame_counts = video_input["video_frame_counts"]
391 + target_dtype = self.embed_vision.embedding_projection.weight.dtype
392 +
393 + if isinstance(frame_counts, torch.Tensor):
394 + fc_list = frame_counts.tolist()
395 + else:
396 + fc_list = list(frame_counts)
397 +
398 + pv_per_video = torch.split(pixel_values, fc_list, dim=0)
399 + pp_per_video = torch.split(pixel_position_ids, fc_list, dim=0)
400 +
401 + per_video_embeddings: list[torch.Tensor] = []
402 + for pv_chunk, pp_chunk in zip(pv_per_video, pp_per_video):
403 + frame_embs: list[torch.Tensor] = []
404 + for i in range(pv_chunk.shape[0]):
405 + pv = pv_chunk[i].unsqueeze(0)
406 + pp = pp_chunk[i].unsqueeze(0)
407 + embedded = self.vision_embedder(pv, pp)
408 + projected = self.embed_vision(embedded.to(target_dtype))
409 + padding_mask = (pp.squeeze(0) == -1).all(dim=-1)
410 + frame_embs.append(projected.squeeze(0)[~padding_mask])
411 + per_video_embeddings.append(torch.cat(frame_embs, dim=0))
412 + return per_video_embeddings
413 +
414 + def _process_audio_input(
415 + self,
416 + audio_input: Gemma4AudioInputs,
417 + ) -> list[torch.Tensor]:
418 + """Project raw waveform-frame features directly to LM space.
419 +
420 + No audio tower: the per-frame raw features are passed straight
421 + through the multimodal embedder, then padding is stripped.
422 + """
423 + input_features = audio_input["input_features_padded"].squeeze(1)
424 + input_features_mask = audio_input["input_features_mask"].squeeze(1)
425 +
426 + target_dtype = self.embed_audio.embedding_projection.weight.dtype
427 + audio_features = self.embed_audio(input_features.to(target_dtype))
428 + per_audio: list[torch.Tensor] = []
429 + for enc, mask in zip(audio_features, input_features_mask, strict=True):
430 + per_audio.append(enc[mask])
431 + return per_audio
432 +
433 + # ------------------------------------------------------------------ #
434 + # Weight loading
435 + # ------------------------------------------------------------------ #
436 +
437 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
438 + ignore_prefixes = [
439 + # Vestigial Gemma3n-style embedding tables not used by
440 + # Gemma4MultimodalEmbedder (which has only projection + norm).
441 + "embed_vision.embedding.",
442 + "embed_audio.embedding.",
443 + ]
444 + if self.embed_audio is None:
445 + ignore_prefixes.append("embed_audio.")
446 +
447 + loader = AutoWeightsLoader(
448 + self,
449 + ignore_unexpected_prefixes=ignore_prefixes,
450 + )
451 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
452 +
453 + # ------------------------------------------------------------------ #
454 + # LoRA / multimodal mapping
455 + # ------------------------------------------------------------------ #
456 +
457 + def get_mm_mapping(self) -> MultiModelKeys:
458 + """Module prefix mapping for the encoder-free model (no towers)."""
459 + connectors = ["embed_vision"]
460 + if self.embed_audio is not None:
461 + connectors.append("embed_audio")
462 + return MultiModelKeys.from_string_field(
463 + language_model="language_model",
464 + connector=connectors,
465 + tower_model=[],
466 + )
vllm/model_executor/models/registry.py
+4
@@ -406,6 +406,10 @@ _MULTIMODAL_MODELS = {
406 "Gemma3nForConditionalGeneration",
407 ),
408 "Gemma4ForConditionalGeneration": ("gemma4_mm", "Gemma4ForConditionalGeneration"),
409 + "Gemma4UnifiedForConditionalGeneration": (
410 + "gemma4_unified",
411 + "Gemma4UnifiedForConditionalGeneration",
412 + ),
413 "GlmAsrForConditionalGeneration": ("glmasr", "GlmAsrForConditionalGeneration"),
414 "GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
415 "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),
vllm/transformers_utils/model_arch_config_convertor.py
+2
@@ -556,6 +556,8 @@ MODEL_ARCH_CONFIG_CONVERTORS = {
556 "gemma4": Gemma4ModelArchConfigConvertor,
557 "gemma4_mtp": Gemma4MTPModelArchConfigConvertor,
558 "gemma4_text": Gemma4ModelArchConfigConvertor,
559 + "gemma4_unified": Gemma4ModelArchConfigConvertor,
560 + "gemma4_unified_text": Gemma4ModelArchConfigConvertor,
561 "glm4_moe_mtp": GLM4MoeMTPModelArchConfigConvertor,
562 "glm_ocr_mtp": GLM4MoeMTPModelArchConfigConvertor,
563 "longcat_flash_mtp": LongCatFlashMTPModelArchConfigConvertor,
vllm/v1/spec_decode/llm_base_proposer.py
+1
@@ -1232,6 +1232,7 @@ class SpecDecodeBaseProposer:
1232 "Qwen3VLForConditionalGeneration",
1233 "Qwen3VLMoeForConditionalGeneration",
1234 "Gemma4ForConditionalGeneration",
1235 + "Gemma4UnifiedForConditionalGeneration",
1236 "Step3p7ForConditionalGeneration",
1237 ]:
1238 self.model.config.image_token_index = target_model.config.image_token_id
vllm/v1/worker/gpu_model_runner.py
+2
@@ -2467,6 +2467,8 @@ class GPUModelRunner(
2467 image_doc_ranges = []
2468 req_state = self.requests[req_id]
2469 for mm_feature in req_state.mm_features:
2470 + if mm_feature.modality == "audio":
2471 + continue
2472 pos_info = mm_feature.mm_position
2473 img_doc_range = pos_info.extract_embeds_range()
2474 for r in img_doc_range: