[Model] Extend device-side mm normalization to GLM4V/GLM5Next (#55389)
Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com>
cjackal committed
Sep 30, 2026 at 16:44 UTC
0a30bc3f9ac3cc1a9339e115377a99d32252aed0
7 files changed
+114
-10
docs/design/mm_processing.md
+3
@@ -80,6 +80,9 @@ GPU-side fusion is controlled by `multimodal_config.mm_device_do_normalize`:
80
81
| name | Architecture | Example HF Models |
82
| -------------------------- | ------------------------------------------ | ----------------------------------------------------- |
83
+| `glm-4v` | `Glm4vForConditionalGeneration` | `zai-org/GLM-4.1V-9B-Thinking`, etc. |
84
+| `glm-5-next` | `Glm5NextForConditionalGeneration` | `zai-org/GLM-5.3-Flash` |
85
+| `glm-ocr` | `GlmOcrForConditionalGeneration` | `zai-org/GLM-OCR` |
86
| `intern-s1-pro` | `InternS1ProForConditionalGeneration` | `internlm/Intern-S1-Pro` |
87
| `intern-s2-mobius` | `InternS2MobiusForConditionalGeneration` | `internlm/Intern-S2-Mobius` |
88
| `llama-nemotron-vl-embed` | `LlamaNemotronVLForEmbedding` | `nvidia/llama-nemotron-embed-vl-1b-v2` |
tests/models/multimodal/generation_ppl_test/test_glm.py
new
+40
@@ -0,0 +1,40 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import pytest
5
+
6
+from tests.models.utils import GenerateModelInfo
7
+
8
+from .ppl_utils import vqa_ppl_test
9
+
10
+MODELS = [
11
+ (GenerateModelInfo("zai-org/GLM-OCR", hf_ppl=13.885052680969238), 28),
12
+]
13
+
14
+
15
+def _pixel_size(factor: int):
16
+ # Match max image token size with qwen's PPL test
17
+ return {
18
+ "size": {
19
+ "shortest_edge": 2 * factor**2,
20
+ "longest_edge": 2 * 1280 * factor**2,
21
+ }
22
+ }
23
+
24
+
25
+@pytest.mark.parametrize("model_info,factor", MODELS)
26
+@pytest.mark.parametrize("mm_device_do_normalize", [True, False])
27
+def test_ppl(
28
+ hf_runner,
29
+ vllm_runner,
30
+ model_info: GenerateModelInfo,
31
+ factor: int,
32
+ mm_device_do_normalize: bool,
33
+):
34
+ vqa_ppl_test(
35
+ hf_runner,
36
+ vllm_runner,
37
+ model_info,
38
+ vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize},
39
+ mm_processor_kwargs=_pixel_size(factor),
40
+ )
tests/models/multimodal/processing/test_glm5next.py
+37
@@ -13,8 +13,12 @@ The checks below are arithmetic: no weights, no GPU.
13
"""
14
15
import pytest
16
+import torch
17
+from PIL import Image
18
19
+from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
20
from vllm.multimodal import MULTIMODAL_REGISTRY
21
+from vllm.platforms import current_platform
22
from vllm.transformers_utils.processors.glm5next import (
23
Glm5NextVideoProcessor,
24
_pixel_budget,
@@ -168,3 +172,36 @@ def test_video_shorter_than_one_sampling_interval_is_rejected(processor):
172
},
173
2,
174
)
175
+
176
+
177
+@pytest.mark.usefixtures("default_vllm_config")
178
+def test_mm_device_do_normalize():
179
+ device = current_platform.device_type
180
+ ctx = build_model_context(
181
+ "zai-org/GLM-5.3-Flash",
182
+ limit_mm_per_prompt={"image": 2},
183
+ )
184
+ assert ctx.model_config.multimodal_config.mm_device_do_normalize
185
+
186
+ ctx.model_config.multimodal_config.mm_device_do_normalize = False
187
+ processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
188
+ images = [
189
+ Image.new("RGB", (310, 470), color=(17, 89, 231)),
190
+ Image.new("RGB", (480, 320), color=(201, 13, 127)),
191
+ ]
192
+ prompt = "<|begin_of_image|><|image|><|end_of_image|>" * len(images)
193
+ mm_items = processor.info.parse_mm_data({"image": images})
194
+
195
+ normalized_inputs = processor(prompt, mm_items=mm_items)
196
+ normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
197
+
198
+ ctx.model_config.multimodal_config.mm_device_do_normalize = True
199
+ raw_inputs = processor(prompt, mm_items=mm_items)
200
+ raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
201
+ assert raw_values.dtype == torch.uint8
202
+
203
+ input_norm = build_mm_input_norm(ctx.model_config).to(device)
204
+ output = input_norm(raw_values.to(device), normalized_values.dtype)
205
+ torch.testing.assert_close(
206
+ output, normalized_values.to(device), rtol=1e-5, atol=1e-6
207
+ )
vllm/model_executor/models/glm4_1v.py
+16
-7
@@ -61,6 +61,10 @@ from vllm.model_executor.layers.attention import (
61
MMEncoderAttention,
62
)
63
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
64
+from vllm.model_executor.layers.fusion.mm_input_norm import (
65
+ IdentityInputNorm,
66
+ build_mm_input_norm,
67
+)
68
from vllm.model_executor.layers.layernorm import RMSNorm
69
from vllm.model_executor.layers.linear import (
70
ColumnParallelLinear,
@@ -616,6 +620,7 @@ class Glm4vVisionTransformer(nn.Module):
620
vision_config: Glm4vVisionConfig,
621
norm_eps: float = 1e-6,
622
quant_config: QuantizationConfig | None = None,
623
+ input_norm: nn.Module | None = None,
624
prefix: str = "",
625
) -> None:
626
super().__init__()
@@ -642,6 +647,7 @@ class Glm4vVisionTransformer(nn.Module):
647
in_channels=in_channels,
648
hidden_size=self.hidden_size,
649
)
650
+ self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
651
652
norm_layer = partial(RMSNorm, eps=norm_eps)
653
head_dim = self.hidden_size // self.num_heads
@@ -924,7 +930,7 @@ class Glm4vVisionTransformer(nn.Module):
930
encoder_metadata = self.prepare_encoder_metadata(grid_thw)
931
932
# patchify
927
- x = x.to(device=self.device, dtype=self.dtype)
933
+ x = self.input_norm(x.to(device=self.device), self.dtype)
934
x = self.patch_embed(x)
935
x = self.post_conv_layernorm(x)
936
@@ -1755,6 +1761,7 @@ class Glm4vForConditionalGeneration(
1761
1762
supports_encoder_tp_data = True
1763
supports_tower_connector_lora = True
1764
+ supports_mm_device_do_normalize = True
1765
1766
@classmethod
1767
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
@@ -1786,6 +1793,7 @@ class Glm4vForConditionalGeneration(
1793
config.vision_config,
1794
norm_eps=getattr(config, "rms_norm_eps", 1e-5),
1795
quant_config=quant_config,
1796
+ input_norm=build_mm_input_norm(self.model_config),
1797
prefix=maybe_prefix(prefix, "visual"),
1798
)
1799
@@ -1867,7 +1875,7 @@ class Glm4vForConditionalGeneration(
1875
if image_input["type"] == "image_embeds":
1876
image_embeds = image_input["image_embeds"].type(self.visual.dtype)
1877
else:
1870
- pixel_values = image_input["pixel_values"].type(self.visual.dtype)
1878
+ pixel_values = image_input["pixel_values"]
1879
if self.use_data_parallel:
1880
return run_dp_sharded_mrope_vision_model(
1881
self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d"
@@ -1888,9 +1896,7 @@ class Glm4vForConditionalGeneration(
1896
if video_input["type"] == "video_embeds":
1897
video_embeds = video_input["video_embeds"].type(self.visual.dtype)
1898
else:
1891
- pixel_values_videos = video_input["pixel_values_videos"].type(
1892
- self.visual.dtype
1893
- )
1899
+ pixel_values_videos = video_input["pixel_values_videos"]
1900
if self.use_data_parallel:
1901
return run_dp_sharded_mrope_vision_model(
1902
self.visual,
@@ -2098,8 +2104,11 @@ class Glm4vForConditionalGeneration(
2104
flattened_patch_size = (
2105
in_channels * temporal_patch_size * patch_size * patch_size
2106
)
2101
- dummy_pixel_values = torch.randn(
2102
- total_patches, flattened_patch_size, device=device, dtype=dtype
2107
+ dummy_pixel_values = torch.zeros(
2108
+ total_patches,
2109
+ flattened_patch_size,
2110
+ device=device,
2111
+ dtype=self.visual.input_norm.input_dtype or dtype,
2112
)
2113
2114
# Override max_seqlen with a safe upper bound for capture.
vllm/model_executor/models/glm_ocr.py
+12
-2
@@ -48,6 +48,7 @@ from vllm.model_executor.layers.attention.mm_encoder_attention import (
48
MMEncoderAttention,
49
)
50
from vllm.model_executor.layers.conv import Conv2dLayer
51
+from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
52
from vllm.model_executor.layers.layernorm import RMSNorm
53
from vllm.model_executor.layers.linear import (
54
QKVParallelLinear,
@@ -254,9 +255,17 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer):
255
vision_config: "GlmOcrVisionConfig",
256
norm_eps: float = 1e-5,
257
quant_config: QuantizationConfig | None = None,
258
+ input_norm: nn.Module | None = None,
259
prefix: str = "",
260
) -> None:
259
- super().__init__(text_config, vision_config, norm_eps, quant_config, prefix)
261
+ super().__init__(
262
+ text_config,
263
+ vision_config,
264
+ norm_eps,
265
+ quant_config,
266
+ input_norm,
267
+ prefix,
268
+ )
269
270
del self.post_conv_layernorm
271
del self.embeddings
@@ -334,7 +343,7 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer):
343
grid_thw = torch.tensor(grid_thw, dtype=torch.int32)
344
345
# patchify
337
- x = x.to(device=self.device, dtype=self.dtype)
346
+ x = self.input_norm(x.to(device=self.device), self.dtype)
347
x = self.patch_embed(x)
348
349
# compute position embedding
@@ -393,5 +402,6 @@ class GlmOcrForConditionalGeneration(Glm4vForConditionalGeneration):
402
config.vision_config,
403
norm_eps=getattr(config, "rms_norm_eps", 1e-5),
404
quant_config=quant_config,
405
+ input_norm=build_mm_input_norm(self.model_config),
406
prefix=maybe_prefix(prefix, "visual"),
407
)
vllm/models/glm5next/common/model.py
+2
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.fused_moe import (
22
GateLinear,
23
fused_moe_make_expert_params_mapping,
24
)
25
+from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
26
from vllm.model_executor.layers.layernorm import RMSNorm
27
from vllm.model_executor.layers.linear import (
28
MergedColumnParallelLinear,
@@ -1103,6 +1104,7 @@ class Glm5NextForConditionalGeneration(
1104
# and yields NaN image features. Mirrors the MLA/KDA proj
1105
# pattern (quant_config=None for BF16 submodules).
1106
quant_config=None,
1107
+ input_norm=build_mm_input_norm(self.model_config),
1108
prefix=maybe_prefix(prefix, "visual"),
1109
)
1110
vllm/models/glm5next/common/multimodal.py
+4
-1
@@ -19,6 +19,7 @@ from vllm.distributed import utils as dist_utils
19
from vllm.model_executor.layers.activation import SiluAndMulWithClamp
20
from vllm.model_executor.layers.attention import MMEncoderAttention
21
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
22
+from vllm.model_executor.layers.fusion.mm_input_norm import IdentityInputNorm
23
from vllm.model_executor.layers.layernorm import RMSNorm
24
from vllm.model_executor.layers.linear import (
25
ColumnParallelLinear,
@@ -353,6 +354,7 @@ class Glm5NextVisionTransformer(nn.Module):
354
vision_config,
355
norm_eps: float = 1e-6,
356
quant_config: QuantizationConfig | None = None,
357
+ input_norm: nn.Module | None = None,
358
prefix: str = "",
359
) -> None:
360
super().__init__()
@@ -386,6 +388,7 @@ class Glm5NextVisionTransformer(nn.Module):
388
in_channels=in_channels,
389
hidden_size=self.hidden_size,
390
)
391
+ self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
392
393
norm_layer = partial(RMSNorm, eps=norm_eps)
394
head_dim = self.hidden_size // self.num_heads
@@ -566,7 +569,7 @@ class Glm5NextVisionTransformer(nn.Module):
569
encoder_metadata: dict[str, torch.Tensor] | None = None,
570
) -> torch.Tensor:
571
# patchify
569
- x = x.to(device=self.device, dtype=self.dtype)
572
+ x = self.input_norm(x.to(device=self.device), self.dtype)
573
x = self.patch_embed(x)
574
575
if encoder_metadata is not None: