@tduncan / Nvidia-Optimization-Stack-VLLM / commits / e3cae8d2ac

[Model] Use upstream GLM-5.3 and Qwen4-Exp configs and processor (#57387)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

Harry Mellor committed Oct 2, 2026 at 03:25 UTC e3cae8d2ac6be8dfd319f3bb8061a1ba4d19bf52
36 files changed +365 -2384
requirements/common.txt
+1 -1
@@ -7,7 +7,7 @@ requests >= 2.33.0
7 tqdm
8 blake3
9 py-cpuinfo
10 -transformers >= 5.10.4
10 +transformers >= 5.16.1
11 huggingface_hub >= 1.31.0 # Required for the `huggingface_hub.utils.httpx` re-export
12 tokenizers >= 0.21.1 # Required for fast incremental detokenization.
13 safetensors >= 0.6.2 # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611
tests/models/glm5next/test_sequence_parallel.py
+1 -1
@@ -15,11 +15,11 @@ from types import SimpleNamespace
15 import pytest
16 import torch
17 from torch import nn
18 +from transformers import Glm5NextTextConfig
19
20 from vllm.model_executor import parameter
21 from vllm.model_executor.layers import linear
22 from vllm.models.glm5next.common import model as glm_model
22 -from vllm.transformers_utils.configs.glm5_next import Glm5NextTextConfig
23
24 HIDDEN, INTERMEDIATE, TP_SIZE = 8, 16, 2
25
tests/models/multimodal/processing/test_glm5next.py
+16 -33
@@ -15,16 +15,15 @@ The checks below are arithmetic: no weights, no GPU.
15 import pytest
16 import torch
17 from PIL import Image
18 +from transformers.models.glm5_next.video_processing_glm5_next import (
19 + Glm5NextVideoProcessor,
20 + smart_resize,
21 +)
22 +from transformers.video_utils import VideoMetadata
23
24 from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
25 from vllm.multimodal import MULTIMODAL_REGISTRY
26 from vllm.platforms import current_platform
22 -from vllm.transformers_utils.processors.glm5next import (
23 - Glm5NextVideoProcessor,
24 - _pixel_budget,
25 - glm_sample_frame_indices,
26 - smart_resize,
27 -)
27
28 from ...utils import build_model_context
29
@@ -48,27 +47,16 @@ def _pixel_path_grid(
47 width: int,
48 ) -> tuple[int, int, int]:
49 """The ``video_grid_thw`` ``Glm5NextVideoProcessor._preprocess`` builds."""
51 - min_pixels, max_pixels = _pixel_budget(
52 - video_processor.min_image_tokens,
53 - video_processor.max_image_tokens,
54 - video_processor.patch_size,
55 - video_processor.merge_size,
56 - video_processor.temporal_patch_size,
57 - )
58 - factor = (
59 - video_processor.patch_size
60 - * video_processor.merge_size
61 - * video_processor.patch_expand_factor
62 - )
50 resized_height, resized_width = smart_resize(
64 - t=num_frames,
65 - h=height,
66 - w=width,
67 - t_factor=video_processor.temporal_patch_size,
68 - h_factor=factor,
69 - w_factor=factor,
70 - min_pixels=min_pixels,
71 - max_pixels=max_pixels,
51 + num_frames=num_frames,
52 + height=height,
53 + width=width,
54 + temporal_factor=video_processor.temporal_patch_size,
55 + factor=video_processor.patch_size
56 + * video_processor.merge_size
57 + * video_processor.patch_expand_factor,
58 + min_pixels=video_processor.min_image_tokens,
59 + max_pixels=video_processor.max_image_tokens,
60 )
61 padded_frames = num_frames + (-num_frames % video_processor.temporal_patch_size)
62 return (
@@ -104,13 +92,8 @@ def test_video_placeholders_match_encoder_rows(
92 info = processor.info
93 video_processor = info.get_video_processor()
94
107 - frame_indices = glm_sample_frame_indices(
108 - total_num_frames,
109 - fps,
110 - duration,
111 - target_fps=video_processor.fps_interval,
112 - max_frame_count=video_processor.max_frame_count_dynamic,
113 - temporal_patch_size=video_processor.temporal_patch_size,
95 + frame_indices = video_processor.sample_frames(
96 + VideoMetadata(total_num_frames=total_num_frames, fps=fps, duration=duration)
97 )
98 grid_t, grid_h, grid_w = _pixel_path_grid(
99 video_processor, len(frame_indices), height, width
tests/models/qwen4_exp/test_config.py
+5 -5
@@ -7,16 +7,13 @@ from unittest.mock import patch
7
8 import pytest
9 import torch
10 +from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
11
12 from vllm.config.speculative import SpeculativeConfig
13 from vllm.model_executor.models.config import (
14 Qwen3_5ForConditionalGenerationConfig,
15 Qwen4ExpForConditionalGenerationConfig,
16 )
16 -from vllm.models.qwen4_exp.config import (
17 - Qwen4ExpConfig,
18 - Qwen4ExpTextConfig,
19 -)
17 from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
18 from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
19
@@ -37,7 +34,10 @@ def _text_config(**kwargs) -> Qwen4ExpTextConfig:
34 "linear_num_value_heads": 2,
35 "linear_key_head_dim": 8,
36 "linear_value_head_dim": 8,
40 - "num_experts": 0,
37 + "num_experts": 4,
38 + "num_experts_per_tok": 2,
39 + # `Qwen4ExpTextConfig` requires an EOS token whenever PLE is enabled.
40 + "eos_token_id": 1,
41 "hc_count": 2,
42 "hc_lowrank": 4,
43 "ple_layer_ids": [1],
tests/models/qwen4_exp/test_ple.py
+1 -1
@@ -10,6 +10,7 @@ import pytest
10 import torch
11 from torch import nn
12 from torch.nn import functional as F
13 +from transformers import Qwen4ExpTextConfig
14
15 import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
16 import vllm.model_executor.parameter as parameter_module
@@ -33,7 +34,6 @@ from vllm.models.qwen4_exp.common.ple import (
34 compute_ple_shard_overlap,
35 copy_ple_embedding_shard_,
36 )
36 -from vllm.models.qwen4_exp.config import Qwen4ExpTextConfig
37 from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
38 Qwen4ExpNGramEmbedding,
39 Qwen4ExpPLEDeviceEmbedding,
tests/multimodal/test_video.py
+3 -15
@@ -1591,10 +1591,6 @@ def test_glm5next_backend_indices_match_sampler(
1591 ):
1592 """The loader must select exactly the frames the processor's sampler
1593 would, with target.fps mapping onto the raw-fps override."""
1594 - from vllm.transformers_utils.processors.glm5next import (
1595 - glm_sample_frame_indices,
1596 - )
1597 -
1594 source = VideoSourceMetadata(
1595 total_frames_num=total_frames, original_fps=original_fps, duration=duration
1596 )
@@ -1604,7 +1600,7 @@ def test_glm5next_backend_indices_match_sampler(
1600 source, target, max_frames=max_frames
1601 )
1602
1607 - assert indices == glm_sample_frame_indices(
1603 + assert indices == Glm5NextVideoBackend._sample_frame_indices(
1604 total_frames,
1605 original_fps,
1606 duration,
@@ -1692,10 +1688,6 @@ def test_glm5next_backend_codec_parity(tmp_path, backend):
1688 if backend == "torchcodec":
1689 pytest.importorskip("torchcodec")
1690
1695 - from vllm.transformers_utils.processors.glm5next import (
1696 - glm_sample_frame_indices,
1697 - )
1698 -
1691 total_frames, fps = 120, 10
1692 path = _write_gray_video(tmp_path, total_frames, fps)
1693 # Dense default sampling (gap 5) and a sparse max_frames cap (gap 20).
@@ -1703,7 +1695,7 @@ def test_glm5next_backend_codec_parity(tmp_path, backend):
1695 kwargs: dict[str, Any] = (
1696 {} if max_frames is None else {"max_frames": max_frames}
1697 )
1706 - expected = glm_sample_frame_indices(
1698 + expected = Glm5NextVideoBackend._sample_frame_indices(
1699 total_frames, float(fps), 12.0, max_frame_count=max_frames
1700 )
1701
@@ -1726,11 +1718,7 @@ def test_glm5next_backend_decodes_only_sampled_frames(tmp_path):
1718 total_frames, fps = 60, 10
1719 path = _write_gray_video(tmp_path, total_frames, fps)
1720
1729 - from vllm.transformers_utils.processors.glm5next import (
1730 - glm_sample_frame_indices,
1731 - )
1732 -
1733 - expected = glm_sample_frame_indices(total_frames, float(fps), 6.0)
1721 + expected = Glm5NextVideoBackend._sample_frame_indices(total_frames, float(fps), 6.0)
1722
1723 frames, metadata = Glm5NextVideoBackend.load_bytes(path.read_bytes())
1724
tests/test_config.py
+44
@@ -4114,3 +4114,47 @@ def test_revision_resolved_for_model(mock_resolve):
4114 assert isinstance(config.revision, ResolvedRevision)
4115 assert config.revision.resolved == REVISION
4116 mock_resolve.assert_any_call(model, None, config.hf_token)
4117 +
4118 +
4119 +@pytest.mark.parametrize(
4120 + ("layer_types", "expected_attention"),
4121 + [
4122 + # Qwen3-Next / Qwen3.5 spell their attention layers "full_attention".
4123 + (["linear_attention", "full_attention"], 1),
4124 + # GLM-5.3-Flash and Qwen4-Exp use sparse attention, which still caches
4125 + # every token and so must count as attention.
4126 + (["linear_attention", "deepseek_sparse_attention"], 1),
4127 + (["linear_attention", "qwen_sparse_attention"], 1),
4128 + (["linear_attention", "linear_attention"], 0),
4129 + ],
4130 +)
4131 +def test_hybrid_layer_counts_sparse_attention_as_attention(
4132 + layer_types, expected_attention
4133 +):
4134 + """Sparse-attention layer types consume a full attention KV cache."""
4135 + model_config = object.__new__(ModelConfig)
4136 + model_config.hf_config = SimpleNamespace()
4137 + model_config.hf_text_config = SimpleNamespace(layer_types=layer_types)
4138 + model_config.model_arch_config = SimpleNamespace(text_model_type="glm5_next")
4139 +
4140 + parallel_config = SimpleNamespace()
4141 + with (
4142 + patch.object(ModelConfig, "is_hybrid", True),
4143 + patch.object(ModelConfig, "has_noops", False),
4144 + patch.object(ModelConfig, "is_attention_free", False),
4145 + patch.object(
4146 + ModelConfig,
4147 + "get_layers_start_end_indices",
4148 + return_value=(0, len(layer_types)),
4149 + ),
4150 + ):
4151 + assert (
4152 + model_config.get_num_layers_by_block_type(parallel_config, "attention")
4153 + == expected_attention
4154 + )
4155 + assert (
4156 + model_config.get_num_layers_by_block_type(
4157 + parallel_config, "linear_attention"
4158 + )
4159 + == len(layer_types) - expected_attention
4160 + )
tests/transformers_utils/processors/test_glm5next.py deleted
-389
@@ -1,389 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""Unit tests for the vLLM-native GLM-5.3-Flash multimodal processor."""
4 -
5 -import math
6 -
7 -import pytest
8 -import torch
9 -from PIL import Image
10 -from transformers.image_utils import PILImageResampling
11 -from transformers.video_utils import VideoMetadata
12 -
13 -from vllm.transformers_utils.processors.glm5next import (
14 - Glm5NextImageProcessor,
15 - Glm5NextVideoProcessor,
16 - _get_pad_content_size,
17 - _resize_or_pad,
18 - glm_sample_frame_indices,
19 - smart_resize,
20 -)
21 -
22 -PATCH_SIZE = 14
23 -MERGE_SIZE = 2
24 -PATCH_EXPAND_FACTOR = 1 # checkpoint processor_config.json
25 -FACTOR = PATCH_SIZE * MERGE_SIZE * PATCH_EXPAND_FACTOR # 28
26 -PIXELS_PER_TOKEN = 2 * (PATCH_SIZE * MERGE_SIZE) ** 2 # 1568
27 -MIN_TOKENS = 16
28 -MAX_TOKENS = 800 # serving-cap equivalent budget, keeps test canvases small
29 -MIN_PIXELS = MIN_TOKENS * PIXELS_PER_TOKEN
30 -MAX_PIXELS = MAX_TOKENS * PIXELS_PER_TOKEN
31 -
32 -
33 -def resize(height: int, width: int, t: int = 2) -> tuple[int, int]:
34 - return smart_resize(
35 - t,
36 - height,
37 - width,
38 - t_factor=2,
39 - h_factor=FACTOR,
40 - w_factor=FACTOR,
41 - min_pixels=MIN_PIXELS,
42 - max_pixels=MAX_PIXELS,
43 - )
44 -
45 -
46 -@pytest.mark.parametrize(
47 - ("height", "width", "t", "expected"),
48 - [
49 - (300, 400, 2, (308, 420)), # ceil alignment, no budget pressure
50 - (50, 80, 2, (112, 168)), # below min budget: proportional upscale
51 - (57, 3, 2, (504, 28)), # slim side ceils to one factor
52 - (2160, 3840, 2, (588, 1064)), # shrink: max canvas under the cap
53 - (112, 11200, 2, (84, 7420)), # extreme aspect stays proportional
54 - (300, 400, 16, (252, 308)), # frame count eats into the budget
55 - (112, 11200, 16, (28, 2800)), # slim side stays positive, budget holds
56 - ],
57 -)
58 -def test_smart_resize_reference_values(height, width, t, expected):
59 - got = smart_resize(
60 - t,
61 - height,
62 - width,
63 - t_factor=2,
64 - h_factor=FACTOR,
65 - w_factor=FACTOR,
66 - min_pixels=MIN_PIXELS,
67 - max_pixels=MAX_PIXELS,
68 - )
69 - assert got == expected
70 - # The aligned canvas never exceeds the budget it was fitted against.
71 - t_bar = max(2, round(t / 2) * 2)
72 - assert t_bar * got[0] * got[1] <= MAX_PIXELS
73 -
74 -
75 -def test_smart_resize_stays_snapped_and_positive():
76 - heights = (1, 27, 57, 111, 113, 300, 720, 2160)
77 - widths = (1, 27, 57, 111, 113, 400, 1280, 11200)
78 - for h in heights:
79 - for w in widths:
80 - for t in (2, 4, 16, 64):
81 - resized_h, resized_w = resize(h, w, t)
82 - assert resized_h > 0 and resized_w > 0
83 - assert resized_h % FACTOR == 0
84 - assert resized_w % FACTOR == 0
85 - t_bar = max(2, round(t / 2) * 2)
86 - assert t_bar * resized_h * resized_w <= MAX_PIXELS
87 -
88 -
89 -@pytest.mark.parametrize(
90 - ("height", "width", "match"),
91 - [
92 - (0, 100, "must be positive"),
93 - (100, 0, "must be positive"),
94 - ],
95 -)
96 -def test_smart_resize_rejects_degenerate_inputs(height, width, match):
97 - with pytest.raises(ValueError, match=match):
98 - resize(height, width, 2)
99 -
100 -
101 -def test_smart_resize_rejects_inverted_budget():
102 - with pytest.raises(ValueError, match="min_pixels must be less than or equal"):
103 - smart_resize(
104 - 2,
105 - 100,
106 - 100,
107 - t_factor=2,
108 - h_factor=FACTOR,
109 - w_factor=FACTOR,
110 - min_pixels=MAX_PIXELS,
111 - max_pixels=MIN_PIXELS,
112 - )
113 -
114 -
115 -def test_get_pad_content_size():
116 - # Oversized content shrinks proportionally, never upscales by default.
117 - assert _get_pad_content_size(300, 400, 308, 420) == (300, 400)
118 - assert _get_pad_content_size(600, 800, 308, 420) == (308, 410)
119 - # allow_upscale enlarges small content toward the canvas.
120 - assert _get_pad_content_size(28, 28, 112, 112, allow_upscale=True) == (112, 112)
121 -
122 -
123 -def test_resize_or_pad_pads_right_and_bottom():
124 - stacked = torch.rand(1, 3, 300, 400)
125 -
126 - def identity_resize(x, size, resample=None):
127 - assert (size.height, size.width) == (300, 400)
128 - return x
129 -
130 - padded = _resize_or_pad(stacked, 308, 420, "pad", None, identity_resize)
131 - assert padded.shape == (1, 3, 308, 420)
132 - torch.testing.assert_close(padded[..., :300, :400], stacked)
133 - assert padded[..., 300:, :].eq(0).all()
134 - assert padded[..., :, 400:].eq(0).all()
135 -
136 - def force_resize(x, size, resample=None):
137 - return torch.zeros(x.shape[0], x.shape[1], size.height, size.width)
138 -
139 - assert _resize_or_pad(stacked, 308, 420, "resize", None, force_resize).shape == (
140 - 1,
141 - 3,
142 - 308,
143 - 420,
144 - )
145 - with pytest.raises(ValueError, match="resize_mode"):
146 - _resize_or_pad(stacked, 308, 420, "crop", None, identity_resize)
147 -
148 -
149 -@pytest.fixture(scope="module")
150 -def image_processor():
151 - return Glm5NextImageProcessor(
152 - min_image_tokens=MIN_TOKENS, max_image_tokens=MAX_TOKENS
153 - )
154 -
155 -
156 -@pytest.mark.parametrize(
157 - ("height", "width"),
158 - [(50, 80), (300, 400), (2160, 3840), (112, 11200)],
159 -)
160 -def test_image_grid_matches_smart_resize(image_processor, height, width):
161 - out = image_processor(Image.new("RGB", (width, height)), return_tensors="pt")
162 - resized_h, resized_w = resize(height, width)
163 - grid = out["image_grid_thw"][0].tolist()
164 - assert grid == [1, resized_h // PATCH_SIZE, resized_w // PATCH_SIZE]
165 - assert out["pixel_values"].shape == (
166 - grid[1] * grid[2],
167 - 3 * 2 * PATCH_SIZE * PATCH_SIZE,
168 - )
169 -
170 -
171 -@pytest.mark.parametrize(
172 - ("height", "width"),
173 - [(50, 80), (300, 400), (2160, 3840), (112, 11200)],
174 -)
175 -def test_image_patch_count_matches_preprocess(image_processor, height, width):
176 - out = image_processor(Image.new("RGB", (width, height)), return_tensors="pt")
177 - grid = out["image_grid_thw"][0].tolist()
178 - expected = grid[1] * grid[2]
179 - assert image_processor.get_number_of_image_patches(height, width) == expected
180 -
181 -
182 -@pytest.fixture(scope="module")
183 -def video_processor():
184 - return Glm5NextVideoProcessor(
185 - min_image_tokens=MIN_TOKENS, max_image_tokens=MAX_TOKENS
186 - )
187 -
188 -
189 -def run_video_preprocess(video_processor, frames, **overrides):
190 - kwargs = dict(
191 - resample=PILImageResampling.BICUBIC,
192 - image_mean=(0.48145466, 0.4578275, 0.40821073),
193 - image_std=(0.26862954, 0.26130258, 0.27577711),
194 - patch_size=PATCH_SIZE,
195 - temporal_patch_size=2,
196 - merge_size=MERGE_SIZE,
197 - patch_expand_factor=PATCH_EXPAND_FACTOR,
198 - return_tensors="pt",
199 - )
200 - kwargs.update(overrides)
201 - return video_processor._preprocess([frames], **kwargs)
202 -
203 -
204 -def test_video_preprocess_pads_odd_frame_count(video_processor):
205 - out = run_video_preprocess(video_processor, torch.rand(7, 3, 300, 400))
206 - grid = out["video_grid_thw"][0].tolist()
207 - # 7 frames padded to 8 -> grid_t 4; canvas ceil-aligned (308, 420).
208 - assert grid == [4, 22, 30]
209 - assert out["pixel_values_videos"].shape == (
210 - 4 * 22 * 30,
211 - 3 * 2 * PATCH_SIZE * PATCH_SIZE,
212 - )
213 -
214 -
215 -def test_video_preprocess_keeps_min_side_under_frame_budget(video_processor):
216 - out = run_video_preprocess(video_processor, torch.rand(16, 3, 112, 11200))
217 - # Budget-fitted canvas (28, 2800): 16 * 28 * 2800 == the pixel cap.
218 - assert out["video_grid_thw"][0].tolist() == [8, 2, 200] # height 28, not 0
219 -
220 -
221 -def test_video_preprocess_pads_content_not_distort(video_processor):
222 - frames = torch.zeros(4, 3, 300, 400)
223 - frames[..., 50, 60] = 1.0 # marker inside the content area
224 - # Bypass rescale/normalize so zero padding stays exactly zero.
225 - out = run_video_preprocess(
226 - video_processor, frames, do_rescale=False, do_normalize=False
227 - )
228 - # Pad mode keeps the 300x400 content aspect on the (308, 420) canvas
229 - # with zero padding on the right/bottom.
230 - grid = out["video_grid_thw"][0].tolist()
231 - assert grid == [2, 22, 30]
232 - patches = out["pixel_values_videos"].view(
233 - grid[0],
234 - grid[1] // MERGE_SIZE,
235 - grid[2] // MERGE_SIZE,
236 - MERGE_SIZE,
237 - MERGE_SIZE,
238 - 3 * 2 * PATCH_SIZE * PATCH_SIZE,
239 - )
240 - frame = patches[0].permute(0, 3, 1, 4, 2).reshape(grid[1], grid[2], -1)
241 - # Patch columns fully right of the 400px content are pure padding.
242 - assert frame[:, math.ceil(400 / PATCH_SIZE) :].abs().max() == 0
243 -
244 -
245 -@pytest.mark.parametrize(
246 - ("total_frames", "fps", "duration", "expected_len", "first", "last"),
247 - [
248 - # Dense source: the tp-scaled greedy overshoots extract_t, so the
249 - # fixup spreads picks uniformly across the whole video (linspace).
250 - (900, 30.0, 30.0, 60, 0, 899),
251 - (3000, 30.0, 100.0, 200, 0, 2999),
252 - # extract_t capped at 2048 frames.
253 - (72000, 30.0, 2400.0, 2048, 0, 71999),
254 - # Low container fps: the greedy picks every frame.
255 - (48, 2.0, 24.0, 48, 0, 47),
256 - # Duration derived from the frame count when metadata lacks it.
257 - (300, 25.0, 0, 26, 0, 299),
258 - ],
259 -)
260 -def test_glm_sample_frame_indices_behaviour(
261 - total_frames, fps, duration, expected_len, first, last
262 -):
263 - indices = glm_sample_frame_indices(total_frames, fps, duration)
264 - assert len(indices) == expected_len
265 - assert indices[0] == first
266 - assert indices[-1] == last
267 - assert len(indices) % 2 == 0
268 - assert indices == sorted(indices)
269 - assert all(0 <= idx < total_frames for idx in indices)
270 -
271 -
272 -def test_glm_sample_frame_indices_short_clip_floor_spread():
273 - # extract_t (20) > total (7): evenly spaced timestamps, deduplicated,
274 - # then pair-padded to an even count.
275 - assert glm_sample_frame_indices(7, 30.0, 10.0) == [0, 1, 2, 3, 4, 5, 6, 6]
276 -
277 -
278 -@pytest.mark.parametrize(
279 - ("kwargs", "expected_len"),
280 - [
281 - ({"target_fps": 0.5}, 16),
282 - ({"max_frame_count": 16}, 16),
283 - ({"target_fps": 8}, 240), # 30s * 8 = 240, under the 2048 cap
284 - ],
285 -)
286 -def test_glm_sample_frame_indices_request_overrides(kwargs, expected_len):
287 - indices = glm_sample_frame_indices(900, 30.0, 30.0, **kwargs)
288 - assert len(indices) == expected_len
289 - assert len(indices) % 2 == 0
290 -
291 -
292 -def test_sample_frames_fps_interval(video_processor):
293 - indices = video_processor.sample_frames(
294 - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0)
295 - )
296 - assert len(indices) == 60
297 - assert indices[0] == 0
298 - assert indices[-1] == 899 # uniform spread reaches the final frame
299 -
300 - # Request overrides reach the sampler through both kwarg spellings. The
301 - # 15-frame spread is odd, so pair-padding duplicates the last frame.
302 - assert (
303 - len(
304 - video_processor.sample_frames(
305 - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
306 - fps=0.5,
307 - )
308 - )
309 - == 16
310 - )
311 - assert (
312 - len(
313 - video_processor.sample_frames(
314 - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
315 - target_fps=0.5,
316 - )
317 - )
318 - == 16
319 - )
320 - assert (
321 - len(
322 - video_processor.sample_frames(
323 - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
324 - max_frames=16,
325 - )
326 - )
327 - == 16
328 - )
329 -
330 -
331 -def test_defaults_mirror_checkpoint_config():
332 - """Bare instantiation matches the checkpoint's ``processor_config.json``
333 - token-budget style defaults; ``size`` carries no budget."""
334 - image_processor = Glm5NextImageProcessor()
335 - assert image_processor.patch_expand_factor == 1
336 - assert image_processor.min_image_tokens == 16
337 - assert image_processor.max_image_tokens == 8000
338 - assert image_processor.resize_mode == "pad"
339 -
340 - video_processor = Glm5NextVideoProcessor()
341 - assert video_processor.patch_expand_factor == 1
342 - assert video_processor.min_image_tokens == 16
343 - assert video_processor.max_image_tokens == 240000
344 - assert video_processor.fps_interval == 2.0
345 - assert video_processor.max_frame_count_dynamic == 2048
346 -
347 -
348 -def test_token_budgets_drive_geometry():
349 - """The token bounds fully determine the pixel budget and the grid."""
350 - token_proc = Glm5NextImageProcessor(min_image_tokens=64, max_image_tokens=512)
351 - img = Image.new("RGB", (400, 300))
352 - grid = token_proc(img, return_tensors="pt")["image_grid_thw"][0].tolist()
353 - # Budgets 100,352..802,816 px: the 300x400 canvas ceils to (308, 420)
354 - # without hitting either bound -> 22x30 patches.
355 - assert grid == [1, 22, 30]
356 - assert token_proc.get_number_of_image_patches(300, 400) == 22 * 30
357 -
358 -
359 -def test_missing_token_budgets_rejected():
360 - proc = Glm5NextImageProcessor(min_image_tokens=None)
361 - with pytest.raises(ValueError, match="min_image_tokens"):
362 - proc(Image.new("RGB", (400, 300)), return_tensors="pt")
363 -
364 -
365 -def test_video_config_fields_land():
366 - """fps_interval / max_frame_count_dynamic from the dedicated config
367 - shape sampling without any request overrides."""
368 - proc = Glm5NextVideoProcessor(
369 - fps_interval=4,
370 - max_frame_count_dynamic=32,
371 - )
372 - indices = proc.sample_frames(
373 - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0)
374 - )
375 - # 30s * 4 = 120 candidates, capped at 32 -> uniform spread of 32.
376 - assert len(indices) == 32
377 - assert indices[0] == 0
378 - assert indices[-1] == 899
379 -
380 - # Request overrides still win over the config values.
381 - assert (
382 - len(
383 - proc.sample_frames(
384 - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
385 - fps=0.5,
386 - )
387 - )
388 - == 16
389 - )
tests/transformers_utils/test_config.py
-53
@@ -24,11 +24,6 @@ from vllm.transformers_utils.config import (
24 try_get_generation_config,
25 uses_mrope,
26 )
27 -from vllm.transformers_utils.configs.glm5_next import (
28 - Glm5NextConfig,
29 - Glm5NextTextConfig,
30 - Glm5NextVisionConfig,
31 -)
27 from vllm.transformers_utils.configs.mistral import adapt_config_dict
28
29
@@ -174,54 +169,6 @@ def test_mistral_yarn_apply_scale_false_disables_yarn_magnitude_scaling():
169 assert config.rope_parameters["attention_factor"] == 1.0
170
171
177 -def test_glm5_next_accepts_deepseek_sparse_attention_layers():
178 - layer_types = ["linear_attention", "deepseek_sparse_attention"]
179 -
180 - config = Glm5NextTextConfig(
181 - num_hidden_layers=len(layer_types), layer_types=layer_types
182 - )
183 -
184 - assert config.layer_types == layer_types
185 - assert config.layers_block_type == ["linear_attention", "attention"]
186 -
187 -
188 -def test_glm5_next_accepts_prebuilt_subconfigs():
189 - text_config = Glm5NextTextConfig(hidden_size=1024)
190 - vision_config = Glm5NextVisionConfig(hidden_size=768)
191 -
192 - config = Glm5NextConfig(
193 - text_config=text_config,
194 - vision_config=vision_config,
195 - )
196 -
197 - assert config.text_config is text_config
198 - assert config.vision_config is vision_config
199 -
200 -
201 -@pytest.mark.parametrize(
202 - ("kwargs", "option"),
203 - [
204 - (
205 - {"index_topk": 2048, "index_dsa_use_layernorm": False},
206 - "index_dsa_use_layernorm",
207 - ),
208 - (
209 - {"index_topk": 2048, "index_kpool_compress": False},
210 - "index_kpool_compress",
211 - ),
212 - (
213 - {"index_topk": 2048, "index_kpool_always_select_tail": False},
214 - "index_kpool_always_select_tail",
215 - ),
216 - ({"hres_vwnstyle": False}, "hres_vwnstyle"),
217 - ({"mhc_no_norm_weight": True}, "mhc_no_norm_weight"),
218 - ],
219 -)
220 -def test_glm5_next_rejects_unimplemented_config_options(kwargs, option):
221 - with pytest.raises(NotImplementedError, match=option):
222 - Glm5NextTextConfig(**kwargs)
223 -
224 -
172 def test_get_llama3_eos_token():
173 model_name = "meta-llama/Llama-3.2-1B-Instruct"
174
tests/v1/spec_decode/test_mtp.py
+1 -1
@@ -43,7 +43,7 @@ def test_glm_mtp_defers_lm_head(default_vllm_config):
43 index_kpool=4,
44 )
45 vllm_config = mock.MagicMock()
46 - vllm_config.speculative_config.draft_model_config.hf_config = config
46 + vllm_config.speculative_config.draft_model_config.hf_text_config = config
47 vllm_config.scheduler_config.max_num_batched_tokens = 4
48
49 with (
vllm/config/model.py
+12 -1
@@ -108,6 +108,16 @@ HfOverrides = dict[str, Any] | Callable[[PreTrainedConfig], PreTrainedConfig]
108 ModelImpl = Literal["auto", "vllm", "transformers", "terratorch"]
109 LayerBlockType = Literal["attention", "linear_attention", "mamba"]
110
111 +_ATTENTION_LAYER_TYPES = frozenset(
112 + {
113 + "full_attention",
114 + "deepseek_sparse_attention",
115 + "qwen_sparse_attention",
116 + }
117 +)
118 +"""`layer_types` spellings that consume a full attention KV cache. Sparse
119 +attention still caches every token, so it counts as attention here."""
120 +
121 _RUNNER_CONVERTS: dict[RunnerType, list[ConvertType]] = {
122 "generate": [],
123 "pooling": ["embed", "classify"],
@@ -1718,7 +1728,8 @@ class ModelConfig:
1728 if layer_types_value is not None:
1729 if block_type == "attention":
1730 return sum(
1721 - t == "full_attention" for t in layer_types_value[start:end]
1731 + t in _ATTENTION_LAYER_TYPES
1732 + for t in layer_types_value[start:end]
1733 )
1734 elif block_type == "linear_attention":
1735 return sum(
vllm/config/speculative.py
+1 -1
@@ -1059,7 +1059,7 @@ class SpeculativeConfig:
1059 )
1060 if hf_config.model_type == "glm5_next":
1061 hf_config.model_type = "glm5_next_mtp"
1062 - n_predict = hf_config.num_nextn_predict_layers
1062 + n_predict = hf_config.get_text_config().num_nextn_predict_layers
1063 hf_config.update(
1064 {"n_predict": n_predict, "architectures": ["Glm5NextMTPModel"]}
1065 )
vllm/models/glm5next/common/attention.py
+18 -14
@@ -4,6 +4,7 @@
4 import torch
5 import torch.nn.functional as F
6 from torch import nn
7 +from transformers import Glm5NextTextConfig
8
9 from vllm.config import (
10 CacheConfig,
@@ -32,7 +33,6 @@ from vllm.model_executor.utils import maybe_disable_graph_partition
33 from vllm.models.glm5next.nvidia.ops.kpool_compress import fwht128_quant_fp8
34 from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool
35 from vllm.platforms import current_platform
35 -from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
36 from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
37 from vllm.utils.math_utils import cdiv, next_power_of_2
38 from vllm.v1.kv_cache_interface import KpoolTailSpec, MLAAttentionSpec
@@ -221,7 +221,7 @@ class Indexer(nn.Module):
221 def __init__(
222 self,
223 vllm_config: VllmConfig,
224 - config: Glm5NextConfig,
224 + config: Glm5NextTextConfig,
225 hidden_size: int,
226 q_lora_rank: int,
227 quant_config: QuantizationConfig | None,
@@ -236,7 +236,7 @@ class Indexer(nn.Module):
236 # self.indexer_cfg = config.attn_module_list_cfg[0]["attn_index"]
237 # Indexer is only constructed for v32 configs, where these sparse-indexer
238 # fields are guaranteed populated; narrow away the `int | None` declared
239 - # on Glm5NextConfig for the optional-indexer case.
239 + # on Glm5NextTextConfig for the optional-indexer case.
240 assert config.index_topk is not None
241 assert config.index_n_heads is not None
242 assert config.index_head_dim is not None
@@ -419,7 +419,7 @@ class Glm5NextMLAAttention(nn.Module):
419 def __init__(
420 self,
421 vllm_config: VllmConfig,
422 - config: Glm5NextConfig,
422 + config: Glm5NextTextConfig,
423 hidden_size: int,
424 num_heads: int,
425 qk_nope_head_dim: int,
@@ -506,28 +506,32 @@ class Glm5NextMLAAttention(nn.Module):
506 prefix=f"{prefix}.o_proj",
507 )
508
509 + # `Glm5NextTextConfig` is NoPE-only and so declares no `rope_parameters`;
510 + # a checkpoint may still ship one.
511 + rope_parameters = getattr(config, "rope_parameters", None)
512 +
513 if not skip_rope:
510 - assert config.rope_parameters is not None
511 - if config.rope_parameters["rope_type"] != "default":
512 - config.rope_parameters["rope_type"] = (
514 + assert rope_parameters is not None
515 + if rope_parameters["rope_type"] != "default":
516 + rope_parameters["rope_type"] = (
517 "deepseek_llama_scaling"
514 - if config.rope_parameters.get("attention_factor") == 1.0
518 + if rope_parameters.get("attention_factor") == 1.0
519 else "deepseek_yarn"
520 )
521
522 self.rotary_emb: RotaryEmbedding | None = get_rope(
523 qk_rope_head_dim,
524 max_position=max_position_embeddings,
521 - rope_parameters=config.rope_parameters,
525 + rope_parameters=rope_parameters,
526 is_neox_style=False,
527 )
528
529 if (
526 - config.rope_parameters["rope_type"] != "default"
527 - and config.rope_parameters["rope_type"] == "deepseek_yarn"
530 + rope_parameters["rope_type"] != "default"
531 + and rope_parameters["rope_type"] == "deepseek_yarn"
532 ):
529 - mscale_all_dim = config.rope_parameters.get("mscale_all_dim", False)
530 - scaling_factor = config.rope_parameters["factor"]
533 + mscale_all_dim = rope_parameters.get("mscale_all_dim", False)
534 + scaling_factor = rope_parameters["factor"]
535 mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
536 self.scaling = self.scaling * mscale * mscale
537 else:
@@ -539,7 +543,7 @@ class Glm5NextMLAAttention(nn.Module):
543 self.indexer_rope_emb: RotaryEmbedding | None = get_rope(
544 qk_rope_head_dim,
545 max_position=max_position_embeddings,
542 - rope_parameters=config.rope_parameters,
546 + rope_parameters=rope_parameters,
547 is_neox_style=not config.indexer_rope_interleave,
548 )
549 # The sparse indexer projects from the MLA q-lora rank, which is
vllm/models/glm5next/common/kda.py
+2 -2
@@ -6,6 +6,7 @@ from dataclasses import replace
6
7 import torch
8 from torch import nn
9 +from transformers import Glm5NextTextConfig
10
11 from vllm.compilation.breakable_cudagraph import eager_break_during_capture
12 from vllm.config import VllmConfig, get_current_vllm_config
@@ -44,7 +45,6 @@ from vllm.model_executor.utils import (
45 )
46 from vllm.platforms import current_platform
47 from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
47 -from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
48 from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
49 from vllm.v1.kv_cache_interface import MambaSpec
50 from vllm.v1.worker.workspace import current_workspace_manager
@@ -201,7 +201,7 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
201
202 def __init__(
203 self,
204 - config: Glm5NextConfig,
204 + config: Glm5NextTextConfig,
205 vllm_config: VllmConfig,
206 prefix: str = "",
207 ) -> None:
vllm/models/glm5next/common/model.py
+74 -26
@@ -6,6 +6,7 @@ from typing import ClassVar, Literal
6
7 import torch
8 from torch import nn
9 +from transformers import Glm5NextTextConfig
10
11 from vllm.config import ParallelConfig, VllmConfig
12 from vllm.distributed import (
@@ -83,7 +84,6 @@ from vllm.models.common.ops.sequence_parallel import (
84 from vllm.multimodal import MULTIMODAL_REGISTRY
85 from vllm.platforms import current_platform
86 from vllm.sequence import IntermediateTensors
86 -from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
87
88 from .attention import Glm5NextMLAAttention
89 from .kda import Glm5NextLinearAttention
@@ -95,6 +95,57 @@ from .multimodal import (
95
96 logger = init_logger(__name__)
97
98 +_MHC_TAU = 0.05
99 +"""mHC routing temperature. A GLM-5.3-Flash trained value that neither the
100 +checkpoint nor `Glm5NextTextConfig` carries."""
101 +
102 +_MHC_POST_MULT_VALUE = 2.0
103 +"""mHC post-multiplier. A GLM-5.3-Flash trained value that neither the
104 +checkpoint nor `Glm5NextTextConfig` carries."""
105 +
106 +_LOGIT_SCALE = 1.0
107 +"""Output logit scale. A GLM-5.3-Flash trained value that neither the
108 +checkpoint nor `Glm5NextTextConfig` carries."""
109 +
110 +_VISION_RMS_NORM_EPS = 1e-6
111 +"""Vision tower RMSNorm epsilon.
112 +
113 +GLM-5.3-Flash checkpoints ship `vision_config.rms_norm_eps = 1e-5`, but the
114 +vision tower was trained with 1e-6. Serving with 1e-5 drifts the RMSNorm and
115 +produces repetitive/degraded image descriptions, so force the trained value
116 +regardless of the checkpoint field.
117 +"""
118 +
119 +
120 +def _is_moe(config: Glm5NextTextConfig) -> bool:
121 + return config.n_routed_experts is not None
122 +
123 +
124 +def _is_kda_layer(config: Glm5NextTextConfig, layer_idx: int) -> bool:
125 + layer_types = config.layer_types
126 + return (
127 + layer_types is not None
128 + and layer_idx < len(layer_types)
129 + and layer_types[layer_idx] == "linear_attention"
130 + )
131 +
132 +
133 +def _is_linear_attn(config: Glm5NextTextConfig) -> bool:
134 + layer_types = config.layer_types
135 + return layer_types is not None and "linear_attention" in layer_types
136 +
137 +
138 +def _validate_supported_config(config: Glm5NextTextConfig) -> None:
139 + """Reject checkpoints using config options this implementation lacks.
140 +
141 + The kpool indexer kernels always keep the incomplete trailing pool, so a
142 + checkpoint asking otherwise would be served silently wrong.
143 + """
144 + if config.index_topk is not None and not config.index_kpool_always_select_tail:
145 + raise NotImplementedError(
146 + "GLM-5.3 sparse indexer requires index_kpool_always_select_tail=True"
147 + )
148 +
149
150 class Glm5NextMLP(nn.Module):
151 def __init__(
@@ -152,7 +203,7 @@ class Glm5NextMLP(nn.Module):
203 class Glm5NextMoE(nn.Module):
204 def __init__(
205 self,
155 - config: Glm5NextConfig,
206 + config: Glm5NextTextConfig,
207 parallel_config: ParallelConfig,
208 quant_config: QuantizationConfig | None = None,
209 prefix: str = "",
@@ -227,10 +278,10 @@ class Glm5NextMoE(nn.Module):
278 shared_experts=self.shared_experts,
279 gate=self.gate,
280 num_experts=config.n_routed_experts,
230 - top_k=config.num_experts_per_token,
281 + top_k=config.num_experts_per_tok,
282 hidden_size=config.hidden_size,
283 intermediate_size=config.moe_intermediate_size,
233 - renormalize=config.moe_renormalize,
284 + renormalize=config.norm_topk_prob,
285 quant_config=quant_config,
286 use_grouped_topk=True,
287 num_expert_group=config.n_group,
@@ -280,7 +331,7 @@ class Glm5NextDecoderLayer(nn.Module):
331 def __init__(
332 self,
333 vllm_config: VllmConfig,
283 - config: Glm5NextConfig,
334 + config: Glm5NextTextConfig,
335 layer_idx: int,
336 prefix: str = "",
337 topk_indices_buffer: torch.Tensor | None = None,
@@ -295,13 +346,13 @@ class Glm5NextDecoderLayer(nn.Module):
346
347 self.hidden_size = config.hidden_size
348 self.layer_idx = layer_idx
298 - self.is_moe = config.is_moe
349 + self.is_moe = _is_moe(config)
350 self.num_hidden_layers = config.num_hidden_layers
351 self.rms_norm_eps = config.rms_norm_eps
352 self.num_experts = config.n_routed_experts
353 self.is_mtp_layer = is_mtp_layer
354 self.mhc = config.mhc
304 - is_kda_layer = not is_mtp_layer and config.is_kda_layer(layer_idx)
355 + is_kda_layer = not is_mtp_layer and _is_kda_layer(config, layer_idx)
356 self.layer_kind = "kda" if is_kda_layer else "mla"
357 self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe
358
@@ -331,7 +382,7 @@ class Glm5NextDecoderLayer(nn.Module):
382 quant_config=None, # MLA projections are BF16 in checkpoint
383 prefix=f"{prefix}.self_attn",
384 topk_indices_buffer=topk_indices_buffer,
334 - skip_rope=config.mla_nope,
385 + skip_rope=config.mla_use_nope,
386 )
387
388 # MTP layers sit past the base model's hidden layers (layer_idx >=
@@ -375,13 +426,13 @@ class Glm5NextDecoderLayer(nn.Module):
426
427 if self.mhc and not is_mtp_layer:
428 # mhc config
378 - self.mhc_num_residual_streams = config.mhc_num_residual_streams
379 - self.mhc_tau = config.mhc_tau
429 + self.mhc_num_residual_streams = config.hc_mult
430 + self.mhc_tau = _MHC_TAU
431 self.hc_eps = config.hc_eps
381 - self.mhc_sinkhorn_iterations = config.mhc_sinkhorn_iterations
382 - self.mhc_post_mult_value = config.mhc_post_mult_value
432 + self.mhc_sinkhorn_iterations = config.hc_sinkhorn_iters
433 + self.mhc_post_mult_value = _MHC_POST_MULT_VALUE
434
384 - n = config.mhc_num_residual_streams
435 + n = config.hc_mult
436 d_model = n * self.hidden_size
437 mix_hc = (2 + n) * n
438
@@ -625,7 +676,8 @@ class Glm5NextModel(nn.Module):
676 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
677 super().__init__()
678
628 - config = vllm_config.model_config.hf_config
679 + config = vllm_config.model_config.hf_text_config
680 + _validate_supported_config(config)
681 self.config = config
682
683 self.vocab_size = config.vocab_size
@@ -768,7 +820,7 @@ class Glm5NextModel(nn.Module):
820 (".in_proj_qkvbfg_a", ".f_a_proj", 4),
821 (".in_proj_qkvbfg_a", ".g_a_proj", 5),
822 ]
771 - if self.config.is_moe:
823 + if _is_moe(self.config):
824 # Params for weights, fp8 weight scales, fp8 activation scales
825 # (param_name, weight_name, expert_id, shard_id)
826 # EPLB: the mapping enumerates physical experts, so it must cover
@@ -798,7 +850,7 @@ class Glm5NextModel(nn.Module):
850 # GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from
851 # ``kv_a_proj_with_mqa``; pad them with zeros for the model shape.
852 kv_a_pad_size = 0
801 - if self.config.mla_nope and self.config.qk_rope_head_dim > 0:
853 + if self.config.mla_use_nope and self.config.qk_rope_head_dim > 0:
854 kv_a_pad_size = self.config.qk_rope_head_dim
855
856 _pending_wk_fp8: dict = {}
@@ -912,7 +964,7 @@ class Glm5NextModel(nn.Module):
964 if (
965 name.endswith(".bias")
966 and name not in params_dict
915 - and not self.config.is_linear_attn
967 + and not _is_linear_attn(self.config)
968 ): # noqa: E501
969 continue
970 # Remapping the name of FP8 kv-scale.
@@ -939,7 +991,7 @@ class Glm5NextForCausalLM(
991 super().__init__()
992 self.model_config = vllm_config.model_config
993 self.vllm_config = vllm_config
942 - self.config = self.model_config.hf_config
994 + self.config = self.model_config.hf_text_config
995 quant_config = vllm_config.quant_config
996 self.quant_config = quant_config
997 self.model = Glm5NextModel(
@@ -955,7 +1007,7 @@ class Glm5NextForCausalLM(
1007 else:
1008 self.lm_head = PPMissingLayer()
1009 self.logits_processor = LogitsProcessor(
958 - self.config.vocab_size, scale=self.config.logit_scale
1010 + self.config.vocab_size, scale=_LOGIT_SCALE
1011 )
1012
1013 def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
@@ -988,7 +1040,7 @@ class Glm5NextForCausalLM(
1040 cls, vllm_config: "VllmConfig"
1041 ) -> tuple[tuple[int, int], tuple[int, int, int]]:
1042 parallel_config = vllm_config.parallel_config
991 - hf_config = vllm_config.model_config.hf_config
1043 + hf_config = vllm_config.model_config.hf_text_config
1044 tp_size = parallel_config.tensor_parallel_size
1045 num_spec = (
1046 vllm_config.speculative_config.num_speculative_tokens
@@ -1092,11 +1144,7 @@ class Glm5NextForConditionalGeneration(
1144 self.visual = Glm5NextVisionTransformer(
1145 config.text_config,
1146 config.vision_config,
1095 - # Read eps from the VISION sub-config, not the top-level
1096 - # `config.rms_norm_eps`: Glm5NextConfig.__getattribute__ mirrors
1097 - # the latter onto text_config (1e-5), silently ignoring the
1098 - # vision tower's own (1e-6) rms_norm_eps.
1099 - norm_eps=config.vision_config.rms_norm_eps,
1147 + norm_eps=_VISION_RMS_NORM_EPS,
1148 # Vision tower ships BF16 weights in this fp8 checkpoint (no
1149 # weight_scale_inv for visual.*), so it must NOT inherit the
1150 # global fp8 quant_config -- doing so incorrectly quantizes
@@ -1166,7 +1214,7 @@ class Glm5NextForConditionalGeneration(
1214
1215
1216 def get_spec_layer_idx_from_weight_name(
1169 - config: Glm5NextConfig, weight_name: str
1217 + config: Glm5NextTextConfig, weight_name: str
1218 ) -> int | None:
1219 if hasattr(config, "num_nextn_predict_layers") and (
1220 config.num_nextn_predict_layers > 0
vllm/models/glm5next/common/mtp.py
+4 -4
@@ -40,7 +40,7 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module):
40 def __init__(self, vllm_config: VllmConfig, prefix: str) -> None:
41 super().__init__()
42 assert vllm_config.speculative_config is not None
43 - config = vllm_config.speculative_config.draft_model_config.hf_config
43 + config = vllm_config.speculative_config.draft_model_config.hf_text_config
44 self.config = config
45
46 self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -114,7 +114,7 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module):
114 class Glm5NextMultiTokenPredictor(nn.Module):
115 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
116 super().__init__()
117 - config = vllm_config.model_config.hf_config
117 + config = vllm_config.model_config.hf_text_config
118 self.mtp_start_layer_idx = config.num_hidden_layers
119 self.num_mtp_layers = config.num_nextn_predict_layers
120 self.layers = torch.nn.ModuleDict(
@@ -211,7 +211,7 @@ class Glm5NextMultiTokenPredictor(nn.Module):
211 class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts):
212 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
213 super().__init__()
214 - self.config = vllm_config.model_config.hf_config
214 + self.config = vllm_config.model_config.hf_text_config
215 self.quant_config = vllm_config.quant_config
216 self.model = Glm5NextMultiTokenPredictor(
217 vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
@@ -313,7 +313,7 @@ class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts):
313 # GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from
314 # ``kv_a_proj_with_mqa``; the FP8-to-BF16 path pads them for the model.
315 kv_a_pad_size = 0
316 - if self.config.mla_nope and self.config.qk_rope_head_dim > 0:
316 + if self.config.mla_use_nope and self.config.qk_rope_head_dim > 0:
317 kv_a_pad_size = self.config.qk_rope_head_dim
318 for name, loaded_weight in weights:
319 if "rotary_emb.inv_freq" in name:
vllm/models/glm5next/common/multimodal.py
+68 -52
@@ -3,13 +3,14 @@
3 """GLM-5.3-Flash vision tower and multimodal processor."""
4
5 from collections.abc import Mapping
6 -from functools import cached_property, partial
6 +from functools import partial
7 from typing import Any
8
9 import numpy as np
10 import torch
11 import torch.nn as nn
12 from einops import rearrange
13 +from transformers.video_utils import VideoMetadata
14
15 from vllm.distributed import (
16 get_tensor_model_parallel_world_size,
@@ -44,7 +45,6 @@ from vllm.model_executor.models.vision import (
45 )
46 from vllm.models.common.ops import fused_q_kv_rmsnorm
47 from vllm.multimodal.parse import ImageSize, MultiModalDataItems
47 -from vllm.transformers_utils.processors.glm5next import glm_sample_frame_indices
48 from vllm.v1.attention.backends.registry import AttentionBackendEnum
49
50
@@ -616,37 +616,48 @@ class Glm5NextVisionTransformer(nn.Module):
616 return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
617
618
619 +_MAX_VIDEO_TOKENS = 30000
620 +
621 +
622 class Glm5NextProcessingInfo(Glm4vProcessingInfo):
620 - """Wires up the vLLM-native processor for the multimodal checkpoint.
621 -
622 - The checkpoint's ``processor_config.json`` declares a custom ``processor_class``
623 - and stores its image/video processor configs inline (no standalone
624 - ``preprocessor_config.json``), so ``AutoProcessor`` cannot resolve the
625 - config. We bypass it and build our own ``Glm5NextProcessor``
626 - (``vllm/transformers_utils/processors/glm5next.py``), a port of the
627 - training-side pipeline that no longer imports transformers' GLM processor
628 - classes. The port applies ``patch_expand_factor`` (checkpoint ships 1)
629 - inside ``smart_resize``'s spatial factor.
623 + """Token-budget geometry for the multimodal checkpoint.
624 +
625 + This checkpoint's ``processor_config.json`` ships the token-budget style
626 + (``min_image_tokens`` / ``max_image_tokens``) with no ``size`` key, so the
627 + inherited Glm4v ``size.longest_edge`` path does not apply.
628 """
629
632 - @cached_property
633 - def _glm5_hf_processor(self):
634 - from vllm.transformers_utils.processors.glm5next import Glm5NextProcessor
630 + def get_hf_processor(self, **kwargs: object):
631 + processor = super().get_hf_processor(**kwargs)
632 + # Cap video inputs to keep encoder profiling from starving the KV cache.
633 + video_processor = processor.video_processor
634 + video_processor.max_image_tokens = min(
635 + video_processor.max_image_tokens, _MAX_VIDEO_TOKENS
636 + )
637 + return processor
638
636 - return Glm5NextProcessor.from_pretrained(self.ctx.model_config.model)
639 + @staticmethod
640 + def _alignment_factor(proc) -> int:
641 + """Spatial factor canvases align to, as ``smart_resize`` takes it."""
642 + return proc.patch_size * proc.merge_size * proc.patch_expand_factor
643
638 - def get_hf_processor(self, **kwargs: object):
639 - return self._glm5_hf_processor
644 + @classmethod
645 + def _pixels_per_token(cls, proc) -> int:
646 + """Pixels one vision token covers, matching ``smart_resize``'s own
647 + ``temporal_factor * factor ** 2``."""
648 + return proc.temporal_patch_size * cls._alignment_factor(proc) ** 2
649
650 def _processor_pixel_budget(self, proc) -> tuple[int, int]:
642 - from vllm.transformers_utils.processors.glm5next import _pixel_budget
643 -
644 - return _pixel_budget(
645 - proc.min_image_tokens,
646 - proc.max_image_tokens,
647 - proc.patch_size,
648 - proc.merge_size,
649 - proc.temporal_patch_size,
651 + """(min_pixels, max_pixels) from the processor's token bounds."""
652 + if proc.min_image_tokens is None or proc.max_image_tokens is None:
653 + raise ValueError(
654 + "min_image_tokens and max_image_tokens must be provided by "
655 + "processor_config.json (or per-call kwargs)."
656 + )
657 + pixels_per_token = self._pixels_per_token(proc)
658 + return (
659 + proc.min_image_tokens * pixels_per_token,
660 + proc.max_image_tokens * pixels_per_token,
661 )
662
663 def _get_image_max_pixels(self) -> int:
@@ -687,19 +698,20 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
698 else:
699 frame_indices = [
700 int(idx)
690 - for idx in glm_sample_frame_indices(
691 - int(metadata.get("total_num_frames", total_frames)),
692 - video_fps,
693 - float(metadata.get("duration") or 0),
694 - target_fps=video_processor.fps_interval,
695 - max_frame_count=video_processor.max_frame_count_dynamic,
696 - temporal_patch_size=temporal_patch_size,
701 + for idx in video_processor.sample_frames(
702 + VideoMetadata(
703 + total_num_frames=int(
704 + metadata.get("total_num_frames", total_frames)
705 + ),
706 + fps=video_fps,
707 + duration=metadata.get("duration"),
708 + )
709 )
710 ]
711 num_frames = len(frame_indices)
712
713 if not frame_indices or num_frames <= 0:
702 - # ``duration * fps_interval < 1`` selects no frames at all; the
714 + # ``duration * fps < 1`` selects no frames at all; the
715 # pixel path raises an opaque IndexError on the same input a moment
716 # later, so say what happened while there is still a request.
717 raise ValueError(
@@ -731,12 +743,13 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
743
744 The inherited Glm4v path resolves the pixel budget from
745 ``size.longest_edge`` and resizes with GLM-4V's ``smart_resize``. This
734 - checkpoint's ``processor_config.json`` ships the token-budget style
735 - (``min_image_tokens`` / ``max_image_tokens``) with no ``size`` key, and
736 - the alignment factor carries ``patch_expand_factor`` — resolve both
737 - from the vLLM-native processor so profiling matches runtime geometry.
746 + checkpoint ships token bounds instead, and the alignment factor carries
747 + ``patch_expand_factor`` — resolve both from the processor so profiling
748 + matches runtime geometry.
749 """
739 - from vllm.transformers_utils.processors.glm5next import smart_resize
750 + from transformers.models.glm5_next.image_processing_glm5_next import (
751 + smart_resize,
752 + )
753
754 vision_config = self.get_hf_config().vision_config
755 patch_size = vision_config.patch_size
@@ -744,22 +757,25 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
757 temporal_patch_size = vision_config.temporal_patch_size
758
759 image_processor = self.get_hf_processor().image_processor
747 - factor = patch_size * merge_size * image_processor.patch_expand_factor
748 - # Keep the profiling search viable when the caller's budget is below
749 - # one aligned canvas of the requested duration.
750 - max_image_pixels = max(max_image_pixels, temporal_patch_size * factor * factor)
760 + factor = self._alignment_factor(image_processor)
761 + temporal_factor = image_processor.temporal_patch_size
762 + # `smart_resize` denominates its bounds in vision tokens. Round down,
763 + # but keep the profiling search viable when the caller's budget is
764 + # below one aligned canvas of the requested duration.
765 + max_image_tokens = max(
766 + max_image_pixels // self._pixels_per_token(image_processor), 1
767 + )
768
769 if do_resize:
753 - t = num_frames if num_frames > temporal_patch_size else temporal_patch_size
770 + t = num_frames if num_frames > temporal_factor else temporal_factor
771 resized_height, resized_width = smart_resize(
755 - t=t,
756 - h=image_height,
757 - w=image_width,
758 - t_factor=temporal_patch_size,
759 - h_factor=factor,
760 - w_factor=factor,
772 + num_frames=t,
773 + height=image_height,
774 + width=image_width,
775 + temporal_factor=temporal_factor,
776 + factor=factor,
777 min_pixels=1,
762 - max_pixels=max_image_pixels,
778 + max_pixels=max_image_tokens,
779 )
780 preprocessed_size = ImageSize(width=resized_width, height=resized_height)
781 else:
@@ -777,7 +793,7 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
793
794
795 class Glm5NextMultiModalProcessor(Glm4vMultiModalProcessor):
780 - """The vLLM-native ``Glm5NextProcessor`` extracts image/video features
796 + """``Glm5NextProcessor`` extracts image/video features
797 only and passes the prompt text through unchanged, so prompt expansion
798 (image token repeat, video frame/timestamp structure) is owned by vLLM's
799 prompt-update machinery — the inherited ``_get_prompt_updates`` builds
vllm/models/qwen4_exp/amd/indexer_qsa.py
+1 -3
@@ -8,6 +8,7 @@ from typing import cast
8
9 import torch
10 from torch import nn
11 +from transformers import Qwen4ExpTextConfig
12
13 from vllm.config import VllmConfig
14 from vllm.forward_context import get_forward_context
@@ -15,9 +16,6 @@ from vllm.model_executor.layers.layernorm import GemmaRMSNorm
16 from vllm.model_executor.layers.linear import ReplicatedLinear
17 from vllm.model_executor.layers.quantization import QuantizationConfig
18 from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
18 -from vllm.transformers_utils.configs.qwen4_exp import (
19 - Qwen4ExpTextConfig,
20 -)
19
20 from ..common.qsa_cache import (
21 QSACompressedKeyCache,
vllm/models/qwen4_exp/amd/model.py
+10 -37
@@ -7,6 +7,7 @@ from itertools import islice
7
8 import torch
9 from torch import nn
10 +from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
11
12 from vllm.compilation.decorators import support_torch_compile
13 from vllm.config import VllmConfig
@@ -47,7 +48,6 @@ from vllm.model_executor.models.qwen3_5 import (
48 )
49 from vllm.model_executor.models.qwen3_next import (
50 Qwen3NextAttention,
50 - Qwen3NextMLP,
51 Qwen3NextSparseMoeBlock,
52 )
53 from vllm.model_executor.models.qwen3_vl import (
@@ -71,13 +71,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
71 from vllm.multimodal.inputs import MultiModalFeatureSpec
72 from vllm.sequence import IntermediateTensors
73 from vllm.tokenizers.registry import cached_tokenizer_from_config
74 -from vllm.transformers_utils.configs.qwen4_exp import (
75 - Qwen4ExpTextConfig,
76 -)
74 from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
75 from vllm.v1.kv_cache_interface import MambaSpec
76
80 -from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig
77 from .hyperconnection import GatedResidual, HyperConnectionConfig
78 from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
79 from .ple_layer import Qwen4ExpPLELayer
@@ -179,8 +175,6 @@ class Qwen4ExpDecoderLayer(nn.Module):
175 ) -> None:
176 super().__init__()
177 config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
182 - model_config = vllm_config.model_config
183 - cache_config = vllm_config.cache_config
178 quant_config = vllm_config.quant_config
179
180 self.config = config
@@ -213,16 +207,13 @@ class Qwen4ExpDecoderLayer(nn.Module):
207 prefix=f"{prefix}.linear_attn",
208 gqa_interleaved_layout=False,
209 )
216 - elif layer_type in ATTENTION_LAYER_TYPES:
217 - use_qsa = (
218 - layer_type == QSA_LAYER_TYPE
219 - or getattr(config, "indexer_n_heads", None) is not None
220 - )
210 + elif layer_type == "qwen_sparse_attention":
211 + use_qsa = getattr(config, "indexer_n_heads", None) is not None
212 if not use_qsa:
213 self.self_attn = Qwen3NextAttention(
214 config,
224 - model_config=model_config,
225 - cache_config=cache_config,
215 + model_config=vllm_config.model_config,
216 + cache_config=vllm_config.cache_config,
217 quant_config=quant_config,
218 prefix=f"{prefix}.self_attn",
219 )
@@ -237,24 +228,9 @@ class Qwen4ExpDecoderLayer(nn.Module):
228 else:
229 raise ValueError(f"Invalid layer_type {layer_type}")
230
240 - mlp_only_layers = getattr(config, "mlp_only_layers", [])
241 - num_experts = getattr(config, "num_experts", 0) or 0
242 - absolute_layer_id = self.layer_idx + 1
243 - is_moe_layer = self.layer_idx not in mlp_only_layers and (
244 - num_experts > 0 and absolute_layer_id % config.decoder_sparse_step == 0
231 + self.mlp = Qwen4ExpSparseMoeBlock(
232 + vllm_config=vllm_config, prefix=f"{prefix}.mlp"
233 )
246 - if is_moe_layer:
247 - self.mlp = Qwen4ExpSparseMoeBlock(
248 - vllm_config=vllm_config, prefix=f"{prefix}.mlp"
249 - )
250 - else:
251 - self.mlp = Qwen3NextMLP(
252 - hidden_size=config.hidden_size,
253 - intermediate_size=config.intermediate_size,
254 - hidden_act=config.hidden_act,
255 - quant_config=quant_config,
256 - prefix=f"{prefix}.mlp",
257 - )
234
235 hc_config = HyperConnectionConfig(
236 hc_count=config.hc_count,
@@ -313,7 +289,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
289
290 if self.layer_type == "linear_attention":
291 attn_out = self.linear_attn(hidden_states=block_input)
316 - elif self.layer_type in ATTENTION_LAYER_TYPES:
292 + elif self.layer_type == "qwen_sparse_attention":
293 attn_out = self.self_attn(
294 hidden_states=block_input,
295 positions=positions,
@@ -404,11 +380,8 @@ class Qwen4ExpModel(nn.Module):
380 self._qsa_layer_ids = frozenset(
381 layer_idx
382 for layer_idx, layer_type in enumerate(config.layer_types)
407 - if layer_type == QSA_LAYER_TYPE
408 - or (
409 - layer_type == "full_attention"
410 - and getattr(config, "indexer_n_heads", None) is not None
411 - )
383 + if layer_type == "qwen_sparse_attention"
384 + and getattr(config, "indexer_n_heads", None) is not None
385 )
386 self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)
387
vllm/models/qwen4_exp/amd/mtp.py
+2 -4
@@ -18,6 +18,7 @@ from collections.abc import Iterable
18 import regex as re
19 import torch
20 from torch import nn
21 +from transformers import Qwen4ExpTextConfig
22
23 from vllm.compilation.decorators import support_torch_compile
24 from vllm.config import VllmConfig, replace, set_current_vllm_config
@@ -45,9 +46,6 @@ from vllm.model_executor.models.utils import (
46 maybe_prefix,
47 )
48 from vllm.sequence import IntermediateTensors
48 -from vllm.transformers_utils.configs.qwen4_exp import (
49 - Qwen4ExpTextConfig,
50 -)
49
50 from .hyperconnection import GatedResidual, HyperConnectionConfig
51 from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
@@ -222,7 +220,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
220 self.layers = nn.ModuleList(
221 Qwen4ExpDecoderLayer(
222 draft_vllm_config,
225 - layer_type="full_attention",
223 + layer_type="qwen_sparse_attention",
224 prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}",
225 )
226 for idx in range(self.num_mtp_layers)
vllm/models/qwen4_exp/amd/ple_layer.py
+1 -3
@@ -8,6 +8,7 @@ from collections.abc import Iterable, Sequence
8 import torch
9 import torch.nn.functional as F
10 from torch import nn
11 +from transformers import Qwen4ExpTextConfig
12
13 from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
14 from vllm.forward_context import get_forward_context
@@ -23,9 +24,6 @@ from vllm.model_executor.layers.quantization.base_config import (
24 QuantizationConfig,
25 )
26 from vllm.model_executor.models.utils import AutoWeightsLoader
26 -from vllm.transformers_utils.configs.qwen4_exp import (
27 - Qwen4ExpTextConfig,
28 -)
27 from vllm.utils.torch_utils import direct_register_custom_op
28 from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
29 from vllm.v1.attention.backends.short_conv_attn import (
vllm/models/qwen4_exp/amd/qsa.py
+1 -3
@@ -8,6 +8,7 @@ from typing import ClassVar, cast
8
9 import torch
10 from torch import nn
11 +from transformers import Qwen4ExpTextConfig
12
13 from vllm.config import VllmConfig
14 from vllm.config.cache import CacheDType
@@ -23,9 +24,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
24 from vllm.model_executor.layers.rotary_embedding import get_rope
25 from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
26 from vllm.platforms import current_platform
26 -from vllm.transformers_utils.configs.qwen4_exp import (
27 - Qwen4ExpTextConfig,
28 -)
27 from vllm.utils.torch_utils import (
28 LayerNameType,
29 _encode_layer_name,
vllm/models/qwen4_exp/config.py deleted
-263
@@ -1,263 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""Qwen4Exp model configuration."""
4 -
5 -from typing import Any, ClassVar, cast
6 -
7 -from transformers import PreTrainedConfig
8 -from transformers.models.qwen3_vl.configuration_qwen3_vl import (
9 - Qwen3VLVisionConfig,
10 -)
11 -
12 -from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
13 -
14 -_QSA_CONFIG_FIELDS = (
15 - "indexer_n_heads",
16 - "indexer_kv_heads",
17 - "indexer_head_dim",
18 - "indexer_budget",
19 - "indexer_compress_ratio",
20 -)
21 -
22 -# Transformers labels Qwen4Exp's sparse-attention layers `qwen_sparse_attention`.
23 -# Older checkpoints label them `full_attention` and mark QSA by setting the
24 -# indexer fields for the whole model.
25 -QSA_LAYER_TYPE = "qwen_sparse_attention"
26 -ATTENTION_LAYER_TYPES = ("full_attention", QSA_LAYER_TYPE)
27 -
28 -
29 -class Qwen4ExpVisionConfig(Qwen3VLVisionConfig):
30 - model_type = "qwen4_exp"
31 - base_config_key = "vision_config"
32 -
33 -
34 -class Qwen4ExpTextConfig(Qwen3NextConfig):
35 - model_type = "qwen4_exp_text"
36 - base_config_key = "text_config"
37 - keys_to_ignore_at_inference = ["past_key_values"]
38 -
39 - def __init__(
40 - self,
41 - hc_count: int = 4,
42 - hc_lowrank: int = 320,
43 - ple_layer_ids: list[int] | None = None,
44 - ple_embed_dim: int | None = None,
45 - ple_conv_kernel_size: int = 4,
46 - ngram_size: int = 3,
47 - heads_per_ngram: int = 8,
48 - ngram_vocab_size_base: int = 20_000_000,
49 - make_ngram_vocab_size_divisible_by: int = 128,
50 - output_gate_type: str = "sigmoid",
51 - rope_parameters: dict[str, Any] | None = None,
52 - layer_types: list[str] | None = None,
53 - **kwargs: Any,
54 - ) -> None:
55 - if hc_count <= 1:
56 - raise ValueError(f"Qwen4Exp requires hc_count > 1, got {hc_count}.")
57 -
58 - if rope_parameters is not None:
59 - if kwargs.get("rope_scaling") is None:
60 - kwargs["rope_scaling"] = rope_parameters
61 - if kwargs.get("rope_theta") is None and "rope_theta" in rope_parameters:
62 - kwargs["rope_theta"] = rope_parameters["rope_theta"]
63 - if (
64 - kwargs.get("partial_rotary_factor") is None
65 - and "partial_rotary_factor" in rope_parameters
66 - ):
67 - kwargs["partial_rotary_factor"] = rope_parameters[
68 - "partial_rotary_factor"
69 - ]
70 -
71 - rope_scaling = kwargs.get("rope_scaling")
72 - rope_theta = kwargs.get("rope_theta", 10_000.0)
73 - super().__init__(layer_types=layer_types, **kwargs)
74 -
75 - normalized_rope_parameters = self.rope_parameters
76 - self.rope_scaling = (
77 - rope_scaling or rope_parameters or normalized_rope_parameters
78 - )
79 - self.rope_parameters = rope_parameters or normalized_rope_parameters
80 - self.rope_theta = rope_theta
81 -
82 - self.hc_count = hc_count
83 - self.hc_lowrank = hc_lowrank
84 - self.ple_layer_ids = ple_layer_ids or []
85 - self.ple_embed_dim = (
86 - self.hidden_size if ple_embed_dim is None else ple_embed_dim
87 - )
88 - self.ple_conv_kernel_size = ple_conv_kernel_size
89 - self.ngram_size = ngram_size
90 - self.heads_per_ngram = heads_per_ngram
91 - self.ngram_vocab_size_base = ngram_vocab_size_base
92 - self.make_ngram_vocab_size_divisible_by = make_ngram_vocab_size_divisible_by
93 - self.output_gate_type = output_gate_type
94 -
95 - self._validate_ple_config()
96 - self._validate_ple_layer_ids()
97 - self._validate_qsa_config()
98 -
99 - def _validate_ple_config(self) -> None:
100 - if self.hc_lowrank <= 0:
101 - raise ValueError(f"hc_lowrank must be positive, got {self.hc_lowrank}")
102 - if self.ngram_size < 2:
103 - raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}")
104 - if self.heads_per_ngram <= 0:
105 - raise ValueError(
106 - f"heads_per_ngram must be positive, got {self.heads_per_ngram}"
107 - )
108 - if self.ple_embed_dim <= 0:
109 - raise ValueError(
110 - f"ple_embed_dim must be positive, got {self.ple_embed_dim}"
111 - )
112 - ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram
113 - if self.ple_embed_dim % ngram_heads:
114 - raise ValueError(
115 - "ple_embed_dim must be divisible by total ngram heads: "
116 - f"{self.ple_embed_dim} % {ngram_heads} != 0"
117 - )
118 - if self.ple_conv_kernel_size <= 0:
119 - raise ValueError(
120 - "ple_conv_kernel_size must be positive, got "
121 - f"{self.ple_conv_kernel_size}"
122 - )
123 - if self.ngram_vocab_size_base <= 0:
124 - raise ValueError("ngram_vocab_size_base must be positive")
125 - if self.make_ngram_vocab_size_divisible_by <= 0:
126 - raise ValueError("make_ngram_vocab_size_divisible_by must be positive")
127 -
128 - def _validate_ple_layer_ids(self) -> None:
129 - invalid = [
130 - layer_id
131 - for layer_id in self.ple_layer_ids
132 - if not 1 <= int(layer_id) <= self.num_hidden_layers
133 - ]
134 - if invalid:
135 - raise ValueError(
136 - "ple_layer_ids are 1-based and must refer to an existing layer; "
137 - f"got {invalid} for {self.num_hidden_layers} layers"
138 - )
139 -
140 - def _validate_qsa_config(self) -> None:
141 - configured = {name: getattr(self, name, None) for name in _QSA_CONFIG_FIELDS}
142 - if all(value is None for value in configured.values()):
143 - if QSA_LAYER_TYPE in (self.layer_types or ()):
144 - raise ValueError(
145 - f"{QSA_LAYER_TYPE} layers require the QSA config fields: "
146 - f"{list(_QSA_CONFIG_FIELDS)}"
147 - )
148 - return
149 -
150 - missing = [name for name, value in configured.items() if value is None]
151 - if missing:
152 - raise ValueError(f"QSA config is missing required fields: {missing}")
153 -
154 - values = {name: int(cast(int, value)) for name, value in configured.items()}
155 - if any(value <= 0 for value in values.values()):
156 - raise ValueError(f"QSA config values must be positive: {values}")
157 - if values["indexer_kv_heads"] != 1:
158 - raise ValueError("the QSA MQA operators require indexer_kv_heads=1")
159 - if values["indexer_budget"] % values["indexer_compress_ratio"] != 0:
160 - raise ValueError(
161 - "indexer_budget must be divisible by indexer_compress_ratio"
162 - )
163 - block_topk = values["indexer_budget"] // values["indexer_compress_ratio"]
164 - if block_topk not in (512, 2048):
165 - raise ValueError(
166 - "QSA requires indexer_budget / indexer_compress_ratio "
167 - f"to be 512 or 2048, got {block_topk}"
168 - )
169 - rotary_dim = int(self.head_dim * self.partial_rotary_factor)
170 - if rotary_dim > values["indexer_head_dim"]:
171 - raise ValueError(
172 - "QSA indexer_head_dim must cover the attention rotary "
173 - f"dimension, got {values['indexer_head_dim']} < {rotary_dim}"
174 - )
175 -
176 - @property
177 - def layers_block_type(self) -> list[str]:
178 - return [
179 - "attention" if layer_type in ATTENTION_LAYER_TYPES else layer_type
180 - for layer_type in self.layer_types
181 - ]
182 -
183 - @property
184 - def short_conv_layer_ids(self) -> list[int]:
185 - if not self.ple_layer_ids:
186 - return []
187 - return sorted({int(layer_id) - 1 for layer_id in self.ple_layer_ids})
188 -
189 - @property
190 - def short_conv_state_shape(self) -> tuple[int, int] | None:
191 - if not self.short_conv_layer_ids:
192 - return None
193 - ple_state_len = (self.ple_conv_kernel_size - 1) * self.ngram_size
194 - ple_channels = self.hidden_size * self.hc_count
195 - return ple_channels, ple_state_len
196 -
197 - @property
198 - def ngram_context_len(self) -> int:
199 - if not self.ple_layer_ids:
200 - return 0
201 - return max(int(self.ngram_size) - 1, 0)
202 -
203 -
204 -class Qwen4ExpConfig(PreTrainedConfig):
205 - model_type = "qwen4_exp"
206 - sub_configs: ClassVar[dict[str, type[PreTrainedConfig]]] = {
207 - "vision_config": Qwen4ExpVisionConfig,
208 - "text_config": Qwen4ExpTextConfig,
209 - }
210 - keys_to_ignore_at_inference = ["past_key_values"]
211 -
212 - def __init__(
213 - self,
214 - text_config: Qwen4ExpTextConfig | dict[str, Any] | None = None,
215 - vision_config: Qwen4ExpVisionConfig | dict[str, Any] | None = None,
216 - image_token_id: int = 248056,
217 - video_token_id: int = 248057,
218 - vision_start_token_id: int = 248053,
219 - vision_end_token_id: int = 248054,
220 - tie_word_embeddings: bool = False,
221 - rope_parameters: dict[str, Any] | None = None,
222 - **kwargs: Any,
223 - ) -> None:
224 - if text_config is not None:
225 - kwargs.pop("split_ngram_parts", None)
226 -
227 - text_kwargs = (
228 - dict(kwargs)
229 - if text_config is None
230 - and "hidden_size" in kwargs
231 - and "num_hidden_layers" in kwargs
232 - else {}
233 - )
234 -
235 - if isinstance(vision_config, dict):
236 - self.vision_config = self.sub_configs["vision_config"](**vision_config)
237 - elif vision_config is None:
238 - self.vision_config = self.sub_configs["vision_config"]()
239 - else:
240 - self.vision_config = vision_config
241 -
242 - if isinstance(text_config, dict):
243 - self.text_config = self.sub_configs["text_config"](**text_config)
244 - elif text_config is None:
245 - self.text_config = self.sub_configs["text_config"](**text_kwargs)
246 - else:
247 - self.text_config = text_config
248 -
249 - self.image_token_id = image_token_id
250 - self.video_token_id = video_token_id
251 - self.vision_start_token_id = vision_start_token_id
252 - self.vision_end_token_id = vision_end_token_id
253 - self.rope_parameters = rope_parameters or getattr(
254 - self.text_config, "rope_parameters", {}
255 - )
256 - super().__init__(**kwargs, tie_word_embeddings=tie_word_embeddings)
257 -
258 -
259 -__all__ = [
260 - "Qwen4ExpConfig",
261 - "Qwen4ExpTextConfig",
262 - "Qwen4ExpVisionConfig",
263 -]
vllm/models/qwen4_exp/nvidia/indexer_qsa.py
+1 -3
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, cast
6
7 import torch
8 from torch import nn
9 +from transformers import Qwen4ExpTextConfig
10
11 from vllm.config import VllmConfig
12 from vllm.forward_context import get_forward_context
@@ -13,9 +14,6 @@ from vllm.model_executor.layers.layernorm import GemmaRMSNorm
14 from vllm.model_executor.layers.linear import ReplicatedLinear
15 from vllm.model_executor.layers.quantization import QuantizationConfig
16 from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
16 -from vllm.transformers_utils.configs.qwen4_exp import (
17 - Qwen4ExpTextConfig,
18 -)
17
18 from ..common.qsa_cache import (
19 QSACompressedKeyCache,
vllm/models/qwen4_exp/nvidia/model.py
+10 -37
@@ -7,6 +7,7 @@ from itertools import islice
7
8 import torch
9 from torch import nn
10 +from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
11
12 from vllm.config import VllmConfig
13 from vllm.distributed import get_pp_group
@@ -46,7 +47,6 @@ from vllm.model_executor.models.qwen3_5 import (
47 )
48 from vllm.model_executor.models.qwen3_next import (
49 Qwen3NextAttention,
49 - Qwen3NextMLP,
50 Qwen3NextSparseMoeBlock,
51 )
52 from vllm.model_executor.models.qwen3_vl import (
@@ -70,13 +70,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
70 from vllm.multimodal.inputs import MultiModalFeatureSpec
71 from vllm.sequence import IntermediateTensors
72 from vllm.tokenizers.registry import cached_tokenizer_from_config
73 -from vllm.transformers_utils.configs.qwen4_exp import (
74 - Qwen4ExpTextConfig,
75 -)
73 from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
74 from vllm.v1.kv_cache_interface import MambaSpec
75
79 -from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig
76 from .hyperconnection import GatedResidual, HyperConnectionConfig
77 from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
78 from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup
@@ -181,8 +177,6 @@ class Qwen4ExpDecoderLayer(nn.Module):
177 ) -> None:
178 super().__init__()
179 config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
184 - model_config = vllm_config.model_config
185 - cache_config = vllm_config.cache_config
180 quant_config = vllm_config.quant_config
181
182 self.config = config
@@ -215,16 +209,13 @@ class Qwen4ExpDecoderLayer(nn.Module):
209 prefix=f"{prefix}.linear_attn",
210 gqa_interleaved_layout=False,
211 )
218 - elif layer_type in ATTENTION_LAYER_TYPES:
219 - use_qsa = (
220 - layer_type == QSA_LAYER_TYPE
221 - or getattr(config, "indexer_n_heads", None) is not None
222 - )
212 + elif layer_type == "qwen_sparse_attention":
213 + use_qsa = getattr(config, "indexer_n_heads", None) is not None
214 if not use_qsa:
215 self.self_attn = Qwen3NextAttention(
216 config,
226 - model_config=model_config,
227 - cache_config=cache_config,
217 + model_config=vllm_config.model_config,
218 + cache_config=vllm_config.cache_config,
219 quant_config=quant_config,
220 prefix=f"{prefix}.self_attn",
221 )
@@ -239,24 +230,9 @@ class Qwen4ExpDecoderLayer(nn.Module):
230 else:
231 raise ValueError(f"Invalid layer_type {layer_type}")
232
242 - mlp_only_layers = getattr(config, "mlp_only_layers", [])
243 - num_experts = getattr(config, "num_experts", 0) or 0
244 - absolute_layer_id = self.layer_idx + 1
245 - is_moe_layer = self.layer_idx not in mlp_only_layers and (
246 - num_experts > 0 and absolute_layer_id % config.decoder_sparse_step == 0
233 + self.mlp = Qwen4ExpSparseMoeBlock(
234 + vllm_config=vllm_config, prefix=f"{prefix}.mlp"
235 )
248 - if is_moe_layer:
249 - self.mlp = Qwen4ExpSparseMoeBlock(
250 - vllm_config=vllm_config, prefix=f"{prefix}.mlp"
251 - )
252 - else:
253 - self.mlp = Qwen3NextMLP(
254 - hidden_size=config.hidden_size,
255 - intermediate_size=config.intermediate_size,
256 - hidden_act=config.hidden_act,
257 - quant_config=quant_config,
258 - prefix=f"{prefix}.mlp",
259 - )
236
237 hc_config = HyperConnectionConfig(
238 hc_count=config.hc_count,
@@ -317,7 +293,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
293
294 if self.layer_type == "linear_attention":
295 attn_out = self.linear_attn(hidden_states=block_input)
320 - elif self.layer_type in ATTENTION_LAYER_TYPES:
296 + elif self.layer_type == "qwen_sparse_attention":
297 attn_out = self.self_attn(
298 hidden_states=block_input,
299 positions=positions,
@@ -397,11 +373,8 @@ class Qwen4ExpModel(nn.Module):
373 self._qsa_layer_ids = frozenset(
374 layer_idx
375 for layer_idx, layer_type in enumerate(config.layer_types)
400 - if layer_type == QSA_LAYER_TYPE
401 - or (
402 - layer_type == "full_attention"
403 - and getattr(config, "indexer_n_heads", None) is not None
404 - )
376 + if layer_type == "qwen_sparse_attention"
377 + and getattr(config, "indexer_n_heads", None) is not None
378 )
379 self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)
380
vllm/models/qwen4_exp/nvidia/mtp.py
+2 -4
@@ -18,6 +18,7 @@ from collections.abc import Iterable
18 import regex as re
19 import torch
20 from torch import nn
21 +from transformers import Qwen4ExpTextConfig
22
23 from vllm.config import VllmConfig, replace, set_current_vllm_config
24 from vllm.distributed import get_pp_group
@@ -44,9 +45,6 @@ from vllm.model_executor.models.utils import (
45 maybe_prefix,
46 )
47 from vllm.sequence import IntermediateTensors
47 -from vllm.transformers_utils.configs.qwen4_exp import (
48 - Qwen4ExpTextConfig,
49 -)
48
49 from .hyperconnection import GatedResidual, HyperConnectionConfig
50 from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
@@ -212,7 +210,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
210 self.layers = nn.ModuleList(
211 Qwen4ExpDecoderLayer(
212 draft_vllm_config,
215 - layer_type="full_attention",
213 + layer_type="qwen_sparse_attention",
214 prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}",
215 )
216 for idx in range(self.num_mtp_layers)
vllm/models/qwen4_exp/nvidia/ngram_embedding.py
+1 -3
@@ -6,6 +6,7 @@ from collections.abc import Iterable
6
7 import torch
8 from torch import nn
9 +from transformers import Qwen4ExpTextConfig
10
11 from vllm.config import get_current_vllm_config
12 from vllm.logger import init_logger
@@ -13,9 +14,6 @@ from vllm.model_executor.layers.quantization.base_config import (
14 QuantizationConfig,
15 )
16 from vllm.model_executor.models.utils import AutoWeightsLoader
16 -from vllm.transformers_utils.configs.qwen4_exp import (
17 - Qwen4ExpTextConfig,
18 -)
17
18 from ..common.ngram_embedding import (
19 Qwen4ExpPLEDeviceEmbedding,
vllm/models/qwen4_exp/nvidia/ple_layer.py
+1 -3
@@ -6,6 +6,7 @@ from collections.abc import Sequence
6
7 import torch
8 from torch import nn
9 +from transformers import Qwen4ExpTextConfig
10
11 from vllm.compilation.breakable_cudagraph import eager_break_during_capture
12 from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
@@ -17,9 +18,6 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
18 MambaStateShapeCalculator,
19 is_conv_state_dim_first,
20 )
20 -from vllm.transformers_utils.configs.qwen4_exp import (
21 - Qwen4ExpTextConfig,
22 -)
21 from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
22 from vllm.v1.attention.backends.short_conv_attn import (
23 PleShortConvAttentionBackend,
vllm/models/qwen4_exp/nvidia/qsa.py
+1 -3
@@ -8,6 +8,7 @@ from typing import ClassVar, cast
8
9 import torch
10 from torch import nn
11 +from transformers import Qwen4ExpTextConfig
12
13 from vllm.compilation.breakable_cudagraph import eager_break_during_capture
14 from vllm.config import VllmConfig
@@ -25,9 +26,6 @@ from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_ro
26 from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
27 from vllm.platforms import current_platform
28 from vllm.platforms.interface import DeviceCapability
28 -from vllm.transformers_utils.configs.qwen4_exp import (
29 - Qwen4ExpTextConfig,
30 -)
29 from vllm.utils.torch_utils import (
30 kv_cache_dtype_str_to_dtype,
31 )
vllm/multimodal/video.py
+83 -12
@@ -714,15 +714,92 @@ class GLM46VVideoBackend(VideoBackend):
714 class Glm5NextVideoBackend(VideoBackend):
715 """GLM-5.3-Flash fps-interval video backend.
716
717 - Selects frames with the same ``glm_sample_frame_indices`` sampler the
718 - processor falls back to, so only the sampled frames are
719 - materialized. ``fps_interval`` semantics (default 2.0) with a
720 - temporal-patch-scaled greedy walk, frame count capped at 2048, temporal
721 - pairs kept even. Request overrides: ``fps`` -> fps interval,
717 + Selects frames with the same sampler the processor falls back to, so only
718 + the sampled frames are materialized. ``fps_interval`` semantics (default
719 + 2.0) with a temporal-patch-scaled greedy walk, frame count capped at 2048,
720 + temporal pairs kept even. Request overrides: ``fps`` -> fps interval,
721 ``max_frames`` -> frame cap, ``temporal_patch_size`` (default 2).
722 """
723
724 _SEEK_GAP_THRESHOLD: ClassVar[int] = 64
725 + _DEFAULT_FPS: ClassVar[float] = 2.0
726 + _DEFAULT_MAX_FRAMES: ClassVar[int] = 2048
727 +
728 + @classmethod
729 + def _sample_frame_indices(
730 + cls,
731 + total_frames: int,
732 + fps: float,
733 + duration: float,
734 + *,
735 + target_fps: float | None = None,
736 + max_frame_count: int | None = None,
737 + temporal_patch_size: int = 2,
738 + ) -> list[int]:
739 + """GLM video frame sampling (training-reference parity).
740 +
741 + ``target_fps`` is the ``fps_interval`` request knob. The greedy walk
742 + advances at ``1 / (temporal_patch_size * target_fps)`` seconds, so on
743 + frame-dense sources it collects more candidates than ``extract_t`` and
744 + the ``> extract_t`` fixup re-spreads the picks uniformly with
745 + ``np.linspace`` -- that fallback is the intended reference behavior, not
746 + an accident. Short clips (fewer frames than ``extract_t``) are spread at
747 + evenly spaced timestamps (``floor`` sampling; the linspace variant
748 + samples frames unevenly and cost 4 points on video grounding evals).
749 + Request overrides: ``target_fps`` -> fps interval, ``max_frame_count``
750 + -> frame cap.
751 + """
752 + max_frame_idx = total_frames - 1
753 + if not duration:
754 + duration = (round(max_frame_idx / fps) + 1) if fps else 0
755 + if max_frame_count is None:
756 + max_frame_count = cls._DEFAULT_MAX_FRAMES
757 + if target_fps is None:
758 + target_fps = cls._DEFAULT_FPS
759 +
760 + extract_t = int(duration * target_fps)
761 + extract_t = min(extract_t, int(max_frame_count))
762 +
763 + duration_per_frame = 1 / fps
764 + timestamps = [i * duration_per_frame for i in range(total_frames)]
765 + max_second = int(duration)
766 +
767 + if total_frames < extract_t:
768 + frame_indices = [
769 + math.floor(_i * total_frames / extract_t) for _i in range(extract_t)
770 + ]
771 + else:
772 + frame_indices = []
773 + current_second = 0.0
774 + inv_fps = 1 / (temporal_patch_size * target_fps)
775 + for frame_index in range(total_frames):
776 + if timestamps[frame_index] >= current_second:
777 + current_second += inv_fps
778 + frame_indices.append(frame_index)
779 + if current_second >= max_second:
780 + break
781 +
782 + if len(frame_indices) < extract_t:
783 + if len(frame_indices) == 0:
784 + start, end = 0, max(total_frames - 1, 0)
785 + else:
786 + start, end = frame_indices[0], frame_indices[-1]
787 + frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()
788 + elif len(frame_indices) > extract_t:
789 + frame_indices = np.linspace(
790 + 0, total_frames - 1, extract_t, dtype=int
791 + ).tolist()
792 +
793 + seen, uniq = set(), []
794 + for idx in frame_indices:
795 + if idx not in seen:
796 + seen.add(idx)
797 + uniq.append(int(idx))
798 +
799 + if len(uniq) & 1:
800 + uniq.append(uniq[-1])
801 +
802 + return uniq
803
804 @classmethod
805 def compute_frames_index_to_sample(
@@ -731,13 +808,7 @@ class Glm5NextVideoBackend(VideoBackend):
808 target: VideoTargetMetadata,
809 **kwargs,
810 ) -> list[int]:
734 - # Lazy import: the processor module sits behind the
735 - # transformers_utils package init, which multimodal must not pull in.
736 - from vllm.transformers_utils.processors.glm5next import (
737 - glm_sample_frame_indices,
738 - )
739 -
740 - return glm_sample_frame_indices(
811 + return cls._sample_frame_indices(
812 source.total_frames_num,
813 source.original_fps,
814 source.duration or 0,
vllm/transformers_utils/config.py
-5
@@ -107,9 +107,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
107 k3_dspark="K3DSparkConfig",
108 funaudiochat="FunAudioChatConfig",
109 granite4_vision="Granite4VisionConfig",
110 - glm5_next="Glm5NextConfig",
111 - glm5_next_text="Glm5NextTextConfig",
112 - glm5_next_vision="Glm5NextVisionConfig",
110 hyperclovax="HyperCLOVAXConfig",
111 hy_v3="HYV3Config",
112 hy_v4="HYV4Config",
@@ -145,8 +142,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
142 qianfan_ocr="QianfanOCRConfig",
143 qwen3_asr="Qwen3ASRConfig",
144 qwen3_next="Qwen3NextConfig",
148 - qwen4_exp="Qwen4ExpConfig",
149 - qwen4_exp_text="Qwen4ExpTextConfig",
145 qwen3_5="Qwen3_5Config",
146 qwen3_5_text="Qwen3_5TextConfig",
147 qwen3_5_moe="Qwen3_5MoeConfig",
vllm/transformers_utils/configs/__init__.py
-12
@@ -43,9 +43,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
43 "FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat",
44 "FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat",
45 "Granite4VisionConfig": "vllm.transformers_utils.configs.granite4_vision",
46 - "Glm5NextConfig": "vllm.transformers_utils.configs.glm5_next",
47 - "Glm5NextTextConfig": "vllm.transformers_utils.configs.glm5_next",
48 - "Glm5NextVisionConfig": "vllm.transformers_utils.configs.glm5_next",
46 "HYV3Config": "vllm.transformers_utils.configs.hy_v3",
47 "HYV4Config": "vllm.transformers_utils.configs.hy_v4",
48 "HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax",
@@ -95,9 +92,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
92 "QianfanOCRVisionConfig": "vllm.transformers_utils.configs.qianfan_ocr",
93 "Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr",
94 "Qwen3NextConfig": "vllm.transformers_utils.configs.qwen3_next",
98 - "Qwen4ExpConfig": "vllm.transformers_utils.configs.qwen4_exp",
99 - "Qwen4ExpTextConfig": "vllm.transformers_utils.configs.qwen4_exp",
100 - "Qwen4ExpVisionConfig": "vllm.transformers_utils.configs.qwen4_exp",
95 "Qwen3_5Config": "vllm.transformers_utils.configs.qwen3_5",
96 "Qwen3_5TextConfig": "vllm.transformers_utils.configs.qwen3_5",
97 "Qwen3_5MoeConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
@@ -140,9 +134,6 @@ __all__ = [
134 "FunAudioChatConfig",
135 "FunAudioChatAudioEncoderConfig",
136 "Granite4VisionConfig",
143 - "Glm5NextConfig",
144 - "Glm5NextTextConfig",
145 - "Glm5NextVisionConfig",
137 "HYV3Config",
138 "HYV4Config",
139 "HyperCLOVAXConfig",
@@ -187,9 +178,6 @@ __all__ = [
178 "QianfanOCRVisionConfig",
179 "Qwen3ASRConfig",
180 "Qwen3NextConfig",
190 - "Qwen4ExpConfig",
191 - "Qwen4ExpTextConfig",
192 - "Qwen4ExpVisionConfig",
181 "Qwen3_5Config",
182 "Qwen3_5TextConfig",
183 "Qwen3_5MoeConfig",
vllm/transformers_utils/configs/glm5_next.py deleted
-420
@@ -1,420 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -from transformers.configuration_utils import PreTrainedConfig
5 -
6 -from vllm.logger import init_logger
7 -
8 -logger = init_logger(__name__)
9 -
10 -
11 -class Glm5NextTextConfig(PreTrainedConfig):
12 - model_type = "glm5_next_text"
13 - base_config_key = "text_config"
14 - keys_to_ignore_at_inference = ["past_key_values"]
15 -
16 - def __init__(
17 - self,
18 - model_type="glm5_next_text",
19 - vocab_size: int = 154880,
20 - hidden_size: int = 4096,
21 - head_dim: int | None = None,
22 - intermediate_size: int = 12288,
23 - num_hidden_layers: int = 45,
24 - num_attention_heads: int = 64,
25 - num_key_value_heads: int | None = None,
26 - hidden_act: str = "silu",
27 - rms_norm_eps: float = 1e-5,
28 - pad_token_id: int | None = 151329,
29 - bos_token_id: int | None = None,
30 - eos_token_id: int | list[int] | None = None,
31 - rope_parameters: dict | None = None,
32 - max_position_embeddings: int = 1048576,
33 - tie_word_embeddings: bool = False,
34 - moe_intermediate_size: int = 2048,
35 - moe_renormalize: bool = True,
36 - scoring_func: str = "sigmoid",
37 - n_routed_experts: int | None = 288,
38 - num_experts_per_token: int = 7,
39 - n_shared_experts: int = 1,
40 - routed_scaling_factor: float = 2.5,
41 - topk_method: str | None = None,
42 - first_k_dense_replace: int = 0,
43 - moe_layer_freq: int = 1,
44 - use_grouped_topk: bool = True,
45 - n_group: int = 1,
46 - topk_group: int = 1,
47 - mla: bool = True,
48 - q_lora_rank: int | None = 1536,
49 - kv_lora_rank: int | None = 512,
50 - qk_nope_head_dim: int = 256,
51 - qk_rope_head_dim: int = 0,
52 - v_head_dim: int | None = 256,
53 - mla_nope: bool | None = True,
54 - num_nextn_predict_layers: int = 1,
55 - # Per-layer layout: "linear_attention" | "deepseek_sparse_attention"
56 - layer_types: list[str] | None = None,
57 - # Per-layer MLP: "dense" | "sparse"
58 - mlp_layer_types: list[str] | None = None,
59 - # Linear-attention (KDA) head config (flattened from the old
60 - # linear_attn_config dict).
61 - linear_head_dim: int = 128,
62 - linear_num_heads: int = 64,
63 - linear_conv_kernel_dim: int = 4,
64 - linear_lower_bound: float = -5.0,
65 - index_head_dim: int | None = None,
66 - index_topk: int | None = None,
67 - index_n_heads: int | None = None,
68 - index_dsa_use_layernorm: bool = True,
69 - index_kpool_compress: bool = True,
70 - # Every ``index_kpool`` indexer K entries pool into one stored entry
71 - # (compress_ratio). The 300B checkpoint ships 4; topk runs at pool
72 - # granularity (select_k = index_topk // index_kpool).
73 - index_kpool: int | None = 4,
74 - index_kpool_always_select_tail: bool = True,
75 - indexer_rope_interleave: bool = False,
76 - mhc: bool | None = True,
77 - mhc_num_residual_streams: int = 4,
78 - hc_eps: float | None = 1e-06,
79 - mhc_tau: float = 0.05,
80 - hres_vwnstyle: bool | None = True,
81 - mhc_no_norm_weight: bool | None = False,
82 - mhc_sinkhorn_iterations: int | None = 20,
83 - mhc_post_mult_value: float | None = 2.0,
84 - swiglu_limit: float | None = None,
85 - logit_scale: float = 1.0,
86 - **kwargs,
87 - ):
88 - # Preserve checkpoint field names and local aliases because their
89 - # consumers use different spellings.
90 - num_experts_per_token = kwargs.get("num_experts_per_tok", num_experts_per_token)
91 - moe_renormalize = kwargs.get("norm_topk_prob", moe_renormalize)
92 - mhc_num_residual_streams = kwargs.get("hc_mult", mhc_num_residual_streams)
93 - mhc_sinkhorn_iterations = kwargs.get(
94 - "hc_sinkhorn_iters", mhc_sinkhorn_iterations
95 - )
96 - # Checkpoint ships ``mla_use_nope`` (not ``mla_nope``); without this
97 - # alias self.mla_nope silently stays at the param default.
98 - mla_nope = kwargs.get("mla_use_nope", mla_nope)
99 - # Checkpoints ship the KDA head config as the ``linear_attn_config``
100 - # dict (head_dim / num_heads / short_conv_kernel_size /
101 - # gate_lower_bound) rather than the flattened top-level fields; fold it
102 - # in so the trained values are read instead of the param defaults
103 - # (which only match this checkpoint by coincidence).
104 - linear_cfg = kwargs.get("linear_attn_config") or {}
105 - if linear_cfg:
106 - linear_head_dim = linear_cfg.get("head_dim", linear_head_dim)
107 - linear_num_heads = linear_cfg.get("num_heads", linear_num_heads)
108 - linear_conv_kernel_dim = linear_cfg.get(
109 - "short_conv_kernel_size", linear_conv_kernel_dim
110 - )
111 - linear_lower_bound = linear_cfg.get("gate_lower_bound", linear_lower_bound)
112 -
113 - if index_topk is not None:
114 - if index_dsa_use_layernorm is not True:
115 - raise NotImplementedError(
116 - "GLM-5.3 sparse indexer requires index_dsa_use_layernorm=True"
117 - )
118 - if index_kpool_compress is not True:
119 - raise NotImplementedError(
120 - "GLM-5.3 sparse indexer requires index_kpool_compress=True"
121 - )
122 - if index_kpool_always_select_tail is not True:
123 - raise NotImplementedError(
124 - "GLM-5.3 sparse indexer requires "
125 - "index_kpool_always_select_tail=True"
126 - )
127 -
128 - if mhc:
129 - if hres_vwnstyle is not True:
130 - raise NotImplementedError("GLM-5.3 mHC requires hres_vwnstyle=True")
131 - if mhc_no_norm_weight not in (False, None):
132 - raise NotImplementedError(
133 - "GLM-5.3 mHC requires mhc_no_norm_weight=False"
134 - )
135 -
136 - self.model_type = model_type
137 - self.vocab_size = vocab_size
138 - self.hidden_size = hidden_size
139 - self.head_dim = (
140 - head_dim if head_dim is not None else hidden_size // num_attention_heads
141 - )
142 - self.intermediate_size = intermediate_size
143 - self.num_hidden_layers = num_hidden_layers
144 - self.num_attention_heads = num_attention_heads
145 -
146 - # for backward compatibility
147 - if num_key_value_heads is None:
148 - num_key_value_heads = num_attention_heads
149 -
150 - self.num_key_value_heads = num_key_value_heads
151 - self.hidden_act = hidden_act
152 - self.rms_norm_eps = rms_norm_eps
153 - self.max_position_embeddings = max_position_embeddings
154 - self.rope_parameters = rope_parameters
155 -
156 - # mla config
157 - self.mla = mla
158 - self.q_lora_rank = q_lora_rank
159 - self.kv_lora_rank = kv_lora_rank
160 - self.qk_nope_head_dim = qk_nope_head_dim
161 - self.qk_rope_head_dim = qk_rope_head_dim
162 - self.v_head_dim = v_head_dim
163 - self.mla_nope = mla_nope
164 - # moe config
165 - self.n_routed_experts = n_routed_experts
166 - self.num_experts_per_token = num_experts_per_token
167 - self.moe_renormalize = moe_renormalize
168 - self.n_shared_experts = n_shared_experts
169 - self.routed_scaling_factor = routed_scaling_factor
170 - self.topk_method = topk_method
171 - self.scoring_func = scoring_func
172 - assert self.scoring_func in ("softmax", "sigmoid")
173 - self.moe_intermediate_size = moe_intermediate_size
174 - self.first_k_dense_replace = first_k_dense_replace
175 - self.moe_layer_freq = moe_layer_freq
176 - self.use_grouped_topk = use_grouped_topk
177 - self.n_group = n_group
178 - self.topk_group = topk_group
179 - self.num_nextn_predict_layers = num_nextn_predict_layers
180 -
181 - # Per-layer attention / MLP layout. Normalize mlp_layer_types from
182 - # first_k_dense_replace when the new-schema field is absent so layer
183 - # construction sees a consistent layout (mirrors cohere2_moe).
184 - self.layer_types = layer_types
185 - if mlp_layer_types is None:
186 - n = self.num_hidden_layers
187 - if first_k_dense_replace is not None:
188 - mlp_layer_types = ["dense"] * first_k_dense_replace + ["sparse"] * (
189 - n - first_k_dense_replace
190 - )
191 - else:
192 - mlp_layer_types = ["sparse"] * n
193 - self.mlp_layer_types = mlp_layer_types
194 -
195 - # Linear-attention (KDA) head config.
196 - self.linear_head_dim = linear_head_dim
197 - self.linear_num_heads = linear_num_heads
198 - self.linear_conv_kernel_dim = linear_conv_kernel_dim
199 - self.linear_lower_bound = linear_lower_bound
200 -
201 - # dsa index config
202 - self.index_head_dim = index_head_dim
203 - self.index_topk = index_topk
204 - self.index_n_heads = index_n_heads
205 - self.index_dsa_use_layernorm = index_dsa_use_layernorm
206 - self.index_kpool_compress = index_kpool_compress
207 - self.index_kpool = index_kpool
208 - self.index_kpool_always_select_tail = index_kpool_always_select_tail
209 - self.indexer_rope_interleave = indexer_rope_interleave
210 -
211 - # mhc config
212 - self.mhc = mhc
213 - self.mhc_num_residual_streams = mhc_num_residual_streams
214 - self.mhc_tau = mhc_tau
215 - self.hres_vwnstyle = hres_vwnstyle
216 - self.hc_eps = hc_eps
217 - self.mhc_no_norm_weight = mhc_no_norm_weight
218 - self.mhc_sinkhorn_iterations = mhc_sinkhorn_iterations
219 - self.mhc_post_mult_value = mhc_post_mult_value
220 -
221 - self.swiglu_limit = swiglu_limit
222 - self.logit_scale = logit_scale
223 -
224 - super().__init__(
225 - pad_token_id=pad_token_id,
226 - bos_token_id=bos_token_id,
227 - eos_token_id=eos_token_id,
228 - tie_word_embeddings=tie_word_embeddings,
229 - **kwargs,
230 - )
231 -
232 - @property
233 - def is_mla(self):
234 - return (
235 - self.q_lora_rank is not None
236 - or self.kv_lora_rank is not None
237 - or self.qk_nope_head_dim is not None
238 - or self.qk_rope_head_dim is not None
239 - or self.v_head_dim is not None
240 - or self.mla_nope is True
241 - )
242 -
243 - @property
244 - def is_moe(self):
245 - return self.n_routed_experts is not None
246 -
247 - @property
248 - def is_linear_attn(self) -> bool:
249 - return self.layer_types is not None and any(
250 - t == "linear_attention" for t in self.layer_types
251 - )
252 -
253 - def is_kda_layer(self, layer_idx: int):
254 - return (
255 - self.layer_types is not None
256 - and layer_idx < len(self.layer_types)
257 - and self.layer_types[layer_idx] == "linear_attention"
258 - )
259 -
260 - @property
261 - def layers_block_type(self):
262 - # Map the schema's per-layer types onto the block strings vLLM's hybrid
263 - # accounting (get_num_layers_by_block_type) recognizes: linear-attention
264 - # layers stay "linear_attention"; every other attention variant collapses
265 - # to "attention".
266 - if self.layer_types is None:
267 - return ["attention"] * self.num_hidden_layers
268 - return [
269 - "linear_attention" if t == "linear_attention" else "attention"
270 - for t in self.layer_types
271 - ]
272 -
273 -
274 -class Glm5NextVisionConfig(PreTrainedConfig):
275 - model_type = "glm5_next_vision"
276 - base_config_key = "vision_config"
277 -
278 - def __init__(
279 - self,
280 - depth: int = 24,
281 - hidden_size: int = 1024,
282 - hidden_act: str = "silu",
283 - image_size: int = 448,
284 - intermediate_size: int = 4096,
285 - num_heads: int = 16,
286 - out_hidden_size: int = 4096,
287 - projection_intermediate_size: int = 10240,
288 - in_channels: int = 3,
289 - initializer_range: float = 0.02,
290 - patch_size: int = 14,
291 - rms_norm_eps: float = 1e-5,
292 - spatial_merge_size: int = 2,
293 - temporal_patch_size: int = 2,
294 - attention_dropout: float = 0.0,
295 - attention_bias: bool = True,
296 - swiglu_limit: float | None = None,
297 - **kwargs,
298 - ):
299 - super().__init__(**kwargs)
300 -
301 - self.depth = depth
302 - self.hidden_size = hidden_size
303 - self.hidden_act = hidden_act
304 - self.image_size = image_size
305 - self.intermediate_size = intermediate_size
306 - self.num_heads = num_heads
307 - self.out_hidden_size = out_hidden_size
308 - # GLM-5.3-Flash merger bottleneck width (absent from the generic
309 - # GLM-OCR vision config); the tower uses it as the PatchMerger
310 - # context_dim instead of text_config.intermediate_size.
311 - self.projection_intermediate_size = projection_intermediate_size
312 - self.in_channels = in_channels
313 - self.initializer_range = initializer_range
314 - self.patch_size = patch_size
315 - # GLM-5.3-Flash checkpoints ship vision_config.rms_norm_eps = 1e-5,
316 - # but the vision tower was trained with 1e-6. Serving with 1e-5 drifts
317 - # the RMSNorm and produces repetitive/degraded image descriptions, so
318 - # force the trained value regardless of the checkpoint field.
319 - self.rms_norm_eps = 1e-6
320 - self.spatial_merge_size = spatial_merge_size
321 - self.temporal_patch_size = temporal_patch_size
322 - self.attention_dropout = attention_dropout
323 - self.attention_bias = attention_bias
324 - self.swiglu_limit = swiglu_limit
325 -
326 -
327 -class Glm5NextConfig(PreTrainedConfig):
328 - model_type = "glm5_next"
329 - sub_configs = {
330 - "vision_config": Glm5NextVisionConfig,
331 - "text_config": Glm5NextTextConfig,
332 - }
333 - keys_to_ignore_at_inference = ["past_key_values"]
334 -
335 - def __init__(
336 - self,
337 - text_config=None,
338 - vision_config=None,
339 - image_token_id: int = 154854,
340 - video_token_id: int = 154855,
341 - image_start_token_id: int = 154830,
342 - image_end_token_id: int = 154831,
343 - video_start_token_id: int = 154832,
344 - video_end_token_id: int = 154833,
345 - **kwargs,
346 - ):
347 - # Init super() first so base-class defaults don't clobber text-config
348 - # values set below (PreTrainedConfig has many text-related defaults
349 - # that differ from Glm5NextTextConfig).
350 - super().__init__(**kwargs)
351 -
352 - if isinstance(vision_config, dict):
353 - self.vision_config = self.sub_configs["vision_config"](**vision_config)
354 - elif vision_config is None:
355 - self.vision_config = self.sub_configs["vision_config"]()
356 - else:
357 - self.vision_config = vision_config
358 -
359 - if isinstance(text_config, dict):
360 - self.text_config = self.sub_configs["text_config"](**text_config)
361 - elif text_config is None:
362 - # Backward compatibility: a flat top-level checkpoint (no nested
363 - # text_config) folds its text fields into Glm5NextTextConfig.
364 - self.text_config = self.sub_configs["text_config"](**kwargs)
365 - else:
366 - self.text_config = text_config
367 -
368 - self.image_token_id = image_token_id
369 - self.video_token_id = video_token_id
370 - self.image_start_token_id = image_start_token_id
371 - self.image_end_token_id = image_end_token_id
372 - self.video_start_token_id = video_start_token_id
373 - self.video_end_token_id = video_end_token_id
374 -
375 - # Mirror attention implementation recursively onto sub-configs.
376 - self._attn_implementation = kwargs.pop("attn_implementation", None)
377 -
378 - # Config-metadata fields that belong to the top-level (multimodal) config
379 - # and must NOT be mirrored onto text_config: ``architectures`` /
380 - # ``torch_dtype`` differ between the top-level config and the text
381 - # sub-config, and mirroring them makes the top-level ``architectures``
382 - # silently read back as None (PreTrainedConfig initializes both to None),
383 - # which then fails model-class resolution ("No model architectures are
384 - # specified").
385 - _UNMIRRORED_KEYS = [
386 - "_name_or_path",
387 - "model_type",
388 - "dtype",
389 - "torch_dtype",
390 - "architectures",
391 - "_attn_implementation_internal",
392 - ]
393 -
394 - def __setattr__(self, key, value):
395 - unmirrored = type(self)._UNMIRRORED_KEYS
396 - if (
397 - (text_config := super().__getattribute__("__dict__").get("text_config"))
398 - is not None
399 - and key not in unmirrored
400 - and key in text_config.__dict__
401 - ):
402 - setattr(text_config, key, value)
403 - else:
404 - super().__setattr__(key, value)
405 -
406 - def __getattribute__(self, key):
407 - unmirrored = type(self)._UNMIRRORED_KEYS
408 - if (
409 - "text_config" in super().__getattribute__("__dict__")
410 - and key not in unmirrored
411 - ):
412 - text_config = super().__getattribute__("text_config")
413 - # Forward both instance attributes AND class-defined properties/
414 - # methods of the text config, so a flat text-only checkpoint
415 - # (model_type "glm5_next", no nested text_config) sees is_moe /
416 - # is_kda_layer / layers_block_type like a Glm5NextTextConfig.
417 - if key in text_config.__dict__ or key in type(text_config).__dict__:
418 - return getattr(text_config, key)
419 -
420 - return super().__getattribute__(key)
vllm/transformers_utils/configs/qwen4_exp.py deleted
-15
@@ -1,15 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""Exports for the package-owned Qwen4Exp configs."""
4 -
5 -from vllm.models.qwen4_exp.config import (
6 - Qwen4ExpConfig,
7 - Qwen4ExpTextConfig,
8 - Qwen4ExpVisionConfig,
9 -)
10 -
11 -__all__ = [
12 - "Qwen4ExpConfig",
13 - "Qwen4ExpTextConfig",
14 - "Qwen4ExpVisionConfig",
15 -]
vllm/transformers_utils/processors/__init__.py
-2
@@ -17,7 +17,6 @@ __all__ = [
17 "FireRedASR2Processor",
18 "FunASRProcessor",
19 "GLM4VProcessor",
20 - "Glm5NextProcessor",
20 "Granite4VisionProcessor",
21 "H2OVLProcessor",
22 "Moondream3Processor",
@@ -56,7 +55,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
55 "FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
56 "FunASRProcessor": "vllm.transformers_utils.processors.funasr",
57 "GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
59 - "Glm5NextProcessor": "vllm.transformers_utils.processors.glm5next",
58 "Granite4VisionProcessor": "vllm.transformers_utils.processors.granite4_vision",
59 "H2OVLProcessor": "vllm.transformers_utils.processors.h2ovl",
60 "InternVLProcessor": "vllm.transformers_utils.processors.internvl",
vllm/transformers_utils/processors/glm5next.py deleted
-953
@@ -1,953 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""vLLM-native multimodal processor for GLM-5.3-Flash."""
4 -
5 -import math
6 -
7 -import numpy as np
8 -import torch
9 -from torchvision.transforms.v2 import functional as tvF
10 -from transformers.image_processing_backends import (
11 - TorchvisionBackend,
12 - group_images_by_shape,
13 - reorder_images,
14 -)
15 -from transformers.image_processing_utils import BatchFeature
16 -from transformers.image_utils import (
17 - OPENAI_CLIP_MEAN,
18 - OPENAI_CLIP_STD,
19 - ChannelDimension,
20 - ImageInput,
21 - PILImageResampling,
22 - SizeDict,
23 - get_image_size,
24 -)
25 -from transformers.models.auto.image_processing_auto import get_image_processor_config
26 -from transformers.processing_utils import (
27 - ImagesKwargs,
28 - MultiModalData,
29 - ProcessingKwargs,
30 - ProcessorMixin,
31 - Unpack,
32 - VideosKwargs,
33 -)
34 -from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
35 -from transformers.utils import TensorType, logging
36 -from transformers.video_processing_utils import BaseVideoProcessor
37 -from transformers.video_utils import (
38 - VideoInput,
39 - VideoMetadata,
40 - group_videos_by_shape,
41 - reorder_videos,
42 -)
43 -
44 -from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
45 -
46 -logger = logging.get_logger(__name__)
47 -
48 -# Cap video inputs at 30,000 vision tokens to keep encoder profiling from
49 -# starving the KV cache. Image inputs retain their checkpoint-defined budget.
50 -_MAX_VIDEO_TOKENS = 30000
51 -
52 -# Frame-sampler fallbacks (mirror the checkpoint's fps_interval=2 /
53 -# max_frame_count_dynamic=2048); used when neither the request nor the
54 -# processor config overrides them.
55 -GLM_VIDEO_DEFAULT_FPS = 2.0
56 -GLM_VIDEO_DEFAULT_MAX_FRAMES = 2048
57 -
58 -
59 -def glm_sample_frame_indices(
60 - total_frames: int,
61 - fps: float,
62 - duration: float,
63 - *,
64 - target_fps: float | None = None,
65 - max_frame_count: int | None = None,
66 - temporal_patch_size: int = 2,
67 -) -> list[int]:
68 - """GLM video frame sampling (training-reference parity).
69 -
70 - ``target_fps`` is the ``fps_interval`` request knob. The greedy walk
71 - advances at ``1 / (temporal_patch_size * target_fps)`` seconds, so on
72 - frame-dense sources it collects more candidates than ``extract_t`` and
73 - the ``> extract_t`` fixup re-spreads the picks uniformly with
74 - ``np.linspace`` -- that fallback is the intended reference behavior, not
75 - an accident. Short clips (fewer frames than ``extract_t``) are spread at
76 - evenly spaced timestamps (``floor`` sampling; the linspace variant
77 - samples frames unevenly and cost 4 points on video grounding evals).
78 - Request overrides: ``target_fps`` -> fps interval, ``max_frame_count``
79 - -> frame cap.
80 - """
81 - max_frame_idx = total_frames - 1
82 - if not duration:
83 - duration = (round(max_frame_idx / fps) + 1) if fps else 0
84 - if max_frame_count is None:
85 - max_frame_count = GLM_VIDEO_DEFAULT_MAX_FRAMES
86 - if target_fps is None:
87 - target_fps = GLM_VIDEO_DEFAULT_FPS
88 -
89 - extract_t = int(duration * target_fps)
90 - extract_t = min(extract_t, int(max_frame_count))
91 -
92 - duration_per_frame = 1 / fps
93 - timestamps = [i * duration_per_frame for i in range(total_frames)]
94 - max_second = int(duration)
95 -
96 - if total_frames < extract_t:
97 - frame_indices = [
98 - math.floor(_i * total_frames / extract_t) for _i in range(extract_t)
99 - ]
100 - else:
101 - frame_indices = []
102 - current_second = 0.0
103 - inv_fps = 1 / (temporal_patch_size * target_fps)
104 - for frame_index in range(total_frames):
105 - if timestamps[frame_index] >= current_second:
106 - current_second += inv_fps
107 - frame_indices.append(frame_index)
108 - if current_second >= max_second:
109 - break
110 -
111 - if len(frame_indices) < extract_t:
112 - if len(frame_indices) == 0:
113 - start, end = 0, max(total_frames - 1, 0)
114 - else:
115 - start, end = frame_indices[0], frame_indices[-1]
116 - frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()
117 - elif len(frame_indices) > extract_t:
118 - frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist()
119 -
120 - seen, uniq = set(), []
121 - for idx in frame_indices:
122 - if idx not in seen:
123 - seen.add(idx)
124 - uniq.append(int(idx))
125 -
126 - if len(uniq) & 1:
127 - uniq.append(uniq[-1])
128 -
129 - return uniq
130 -
131 -
132 -def _ceil_to_factor(value: int, factor: int) -> int:
133 - """Round a positive integer upward to the nearest multiple of factor."""
134 - return math.ceil(value / factor) * factor
135 -
136 -
137 -def _fit_aligned_size_within_budget(
138 - t: int,
139 - h: int,
140 - w: int,
141 - h_factor: int,
142 - w_factor: int,
143 - max_pixels: int,
144 -) -> tuple[int, int]:
145 - """Largest proportional size whose upward-aligned canvas fits the budget.
146 -
147 - Binary search on the unaligned content height; each candidate is rounded
148 - upward to h_factor/w_factor, so the returned canvas always satisfies
149 - ``t * aligned_h * aligned_w <= max_pixels``.
150 - """
151 - minimum_pixels = t * h_factor * w_factor
152 - if max_pixels < minimum_pixels:
153 - raise ValueError(
154 - f"max_pixels={max_pixels} is too small. At least "
155 - f"{minimum_pixels} pixels are required for one aligned patch."
156 - )
157 -
158 - low, high = 1, h
159 - best_h, best_w = h_factor, w_factor
160 - while low <= high:
161 - content_h = (low + high) // 2
162 - content_w = max(1, math.floor(w * content_h / h))
163 - aligned_h = _ceil_to_factor(content_h, h_factor)
164 - aligned_w = _ceil_to_factor(content_w, w_factor)
165 - if t * aligned_h * aligned_w <= max_pixels:
166 - best_h, best_w = aligned_h, aligned_w
167 - low = content_h + 1
168 - else:
169 - high = content_h - 1
170 - return best_h, best_w
171 -
172 -
173 -def smart_resize(
174 - t: int,
175 - h: int,
176 - w: int,
177 - t_factor: int = 1,
178 - h_factor: int = 28,
179 - w_factor: int = 28,
180 - min_pixels: int = 56 * 56,
181 - max_pixels: int = 14 * 14 * 4 * 1280,
182 -) -> tuple[int, int]:
183 - """GLM-5.3-Flash ``smart_resize``: upward-aligned canvas under a
184 - ``t_bar * h_bar * w_bar`` pixel budget.
185 -
186 - Height/width always round UP to their factors (content is then padded,
187 - never cropped or distorted); an over-budget canvas is refit by binary
188 - search instead of one-shot square-root scaling. ``h_factor`` /
189 - ``w_factor`` carry ``patch_expand_factor`` on top of
190 - ``patch_size * merge_size``; ``t_factor`` is ``temporal_patch_size``. For
191 - a still image ``t = t_factor = temporal_patch_size`` so ``t_bar =
192 - temporal_patch_size``.
193 - """
194 - if min(t, h, w, t_factor, h_factor, w_factor) <= 0:
195 - raise ValueError("Image dimensions and alignment factors must be positive.")
196 - if min_pixels <= 0 or max_pixels <= 0:
197 - raise ValueError("min_pixels and max_pixels must be positive.")
198 - if min_pixels > max_pixels:
199 - raise ValueError("min_pixels must be less than or equal to max_pixels.")
200 -
201 - t_bar = max(t_factor, round(t / t_factor) * t_factor)
202 - h_bar = _ceil_to_factor(h, h_factor)
203 - w_bar = _ceil_to_factor(w, w_factor)
204 -
205 - if t_bar * h_bar * w_bar > max_pixels:
206 - h_bar, w_bar = _fit_aligned_size_within_budget(
207 - t=t_bar,
208 - h=h,
209 - w=w,
210 - h_factor=h_factor,
211 - w_factor=w_factor,
212 - max_pixels=max_pixels,
213 - )
214 - elif t_bar * h_bar * w_bar < min_pixels:
215 - beta = math.sqrt(min_pixels / (t * h * w))
216 - h_bar = _ceil_to_factor(max(1, math.ceil(h * beta)), h_factor)
217 - w_bar = _ceil_to_factor(max(1, math.ceil(w * beta)), w_factor)
218 -
219 - # Alignment can push a candidate slightly over a tight max_pixels
220 - # budget. Refit it when that happens.
221 - if t_bar * h_bar * w_bar > max_pixels:
222 - h_bar, w_bar = _fit_aligned_size_within_budget(
223 - t=t_bar,
224 - h=h,
225 - w=w,
226 - h_factor=h_factor,
227 - w_factor=w_factor,
228 - max_pixels=max_pixels,
229 - )
230 -
231 - return h_bar, w_bar
232 -
233 -
234 -def _get_pad_content_size(
235 - image_height: int,
236 - image_width: int,
237 - canvas_height: int,
238 - canvas_width: int,
239 - allow_upscale: bool = False,
240 -) -> tuple[int, int]:
241 - """Aspect-ratio-preserving content size that fits the canvas.
242 -
243 - Oversized images are shrunk proportionally. Small images are enlarged
244 - only when ``allow_upscale``. Padding is applied after the resize.
245 - """
246 - scale = min(canvas_height / image_height, canvas_width / image_width)
247 - if not allow_upscale:
248 - scale = min(1.0, scale)
249 - content_height = max(1, min(canvas_height, math.floor(image_height * scale)))
250 - content_width = max(1, min(canvas_width, math.floor(image_width * scale)))
251 - return content_height, content_width
252 -
253 -
254 -def _resize_or_pad(
255 - stacked_images: torch.Tensor,
256 - target_height: int,
257 - target_width: int,
258 - resize_mode: str,
259 - resample: "PILImageResampling | tvF.InterpolationMode | int | None",
260 - resize,
261 - allow_upscale: bool = False,
262 -) -> torch.Tensor:
263 - """Resize onto the aligned canvas, or keep the aspect ratio and
264 - zero-pad the right/bottom sides (``resize_mode="pad"``)."""
265 - height, width = stacked_images.shape[-2:]
266 -
267 - if resize_mode == "resize":
268 - return resize(
269 - stacked_images,
270 - size=SizeDict(height=target_height, width=target_width),
271 - resample=resample,
272 - )
273 -
274 - if resize_mode != "pad":
275 - raise ValueError("resize_mode must be either 'resize' or 'pad'.")
276 -
277 - content_height, content_width = _get_pad_content_size(
278 - image_height=height,
279 - image_width=width,
280 - canvas_height=target_height,
281 - canvas_width=target_width,
282 - allow_upscale=allow_upscale,
283 - )
284 -
285 - if (content_height, content_width) != (height, width):
286 - stacked_images = resize(
287 - stacked_images,
288 - size=SizeDict(height=content_height, width=content_width),
289 - resample=resample,
290 - )
291 -
292 - # torchvision padding order: [left, top, right, bottom] -> pad only the
293 - # right and bottom sides.
294 - return tvF.pad(
295 - stacked_images,
296 - padding=[0, 0, target_width - content_width, target_height - content_height],
297 - fill=0,
298 - )
299 -
300 -
301 -def _pixel_budget(
302 - min_image_tokens: int | None,
303 - max_image_tokens: int | None,
304 - patch_size: int,
305 - merge_size: int,
306 - temporal_patch_size: int,
307 -) -> tuple[int, int]:
308 - """(min_pixels, max_pixels) from the token bounds of
309 - ``processor_config.json``; one vision token covers
310 - ``temporal_patch_size * (patch_size * merge_size) ** 2`` pixels."""
311 - if min_image_tokens is None or max_image_tokens is None:
312 - raise ValueError(
313 - "min_image_tokens and max_image_tokens must be provided by "
314 - "processor_config.json (or per-call kwargs)."
315 - )
316 - factor = temporal_patch_size * (patch_size * merge_size) ** 2
317 - return min_image_tokens * factor, max_image_tokens * factor
318 -
319 -
320 -class Glm5NextImageProcessorKwargs(ImagesKwargs, total=False): # type: ignore[call-arg]
321 - patch_size: int | None
322 - temporal_patch_size: int | None
323 - merge_size: int | None
324 - patch_expand_factor: int | None
325 - resize_mode: str | None
326 - min_image_tokens: int | None
327 - max_image_tokens: int | None
328 -
329 -
330 -class Glm5NextImageProcessor(TorchvisionBackend):
331 - """Fast torchvision image processor for GLM-5.3-Flash.
332 -
333 - ``patch_expand_factor`` multiplies into the ``smart_resize`` spatial
334 - factor ``patch_size * merge_size``. ``resize_mode`` picks the geometry:
335 - ``"pad"`` (default) preserves the aspect ratio and zero-pads the
336 - right/bottom of the upward-aligned canvas, ``"resize"`` stretches onto
337 - it. Defaults mirror the checkpoint's ``image_processor`` config.
338 - """
339 -
340 - do_resize = True
341 - resample = PILImageResampling.BICUBIC
342 - size = {"longest_edge": 1} # unused: budgets come from the token bounds
343 - do_rescale = True
344 - do_normalize = True
345 - image_mean = OPENAI_CLIP_MEAN
346 - image_std = OPENAI_CLIP_STD
347 - do_convert_rgb = True
348 - patch_size = 14
349 - temporal_patch_size = 2
350 - merge_size = 2
351 - patch_expand_factor = 1
352 - resize_mode = "pad"
353 - min_image_tokens = 16
354 - max_image_tokens = 8000
355 - valid_kwargs = Glm5NextImageProcessorKwargs
356 - model_input_names = ["pixel_values", "image_grid_thw"]
357 -
358 - def _preprocess(
359 - self,
360 - images: list[torch.Tensor],
361 - do_resize: bool,
362 - size: SizeDict,
363 - resample: "PILImageResampling | tvF.InterpolationMode | int | None",
364 - do_rescale: bool,
365 - rescale_factor: float,
366 - do_normalize: bool,
367 - image_mean: float | list[float] | None,
368 - image_std: float | list[float] | None,
369 - patch_size: int,
370 - temporal_patch_size: int,
371 - merge_size: int,
372 - patch_expand_factor: int,
373 - resize_mode: str | None,
374 - min_image_tokens: int | None,
375 - max_image_tokens: int | None,
376 - disable_grouping: bool | None,
377 - return_tensors: str | TensorType | None,
378 - **kwargs,
379 - ) -> BatchFeature:
380 - resize_mode = resize_mode if resize_mode is not None else self.resize_mode
381 - min_pixels, max_pixels = _pixel_budget(
382 - min_image_tokens if min_image_tokens is not None else self.min_image_tokens,
383 - max_image_tokens if max_image_tokens is not None else self.max_image_tokens,
384 - patch_size,
385 - merge_size,
386 - temporal_patch_size,
387 - )
388 - grouped_images, grouped_images_index = group_images_by_shape(
389 - images, disable_grouping=disable_grouping
390 - )
391 - resized_images_grouped = {}
392 - for shape, stacked_images in grouped_images.items():
393 - height, width = stacked_images.shape[-2:]
394 - if do_resize:
395 - resized_height, resized_width = smart_resize(
396 - t=temporal_patch_size,
397 - h=height,
398 - w=width,
399 - t_factor=temporal_patch_size,
400 - h_factor=patch_size * merge_size * patch_expand_factor,
401 - w_factor=patch_size * merge_size * patch_expand_factor,
402 - min_pixels=min_pixels,
403 - max_pixels=max_pixels,
404 - )
405 - stacked_images = _resize_or_pad(
406 - stacked_images,
407 - target_height=resized_height,
408 - target_width=resized_width,
409 - resize_mode=resize_mode,
410 - resample=resample,
411 - resize=self.resize,
412 - allow_upscale=(temporal_patch_size * height * width < min_pixels),
413 - )
414 - resized_images_grouped[shape] = stacked_images
415 -
416 - resized_images = reorder_images(resized_images_grouped, grouped_images_index)
417 -
418 - grouped_images, grouped_images_index = group_images_by_shape(
419 - resized_images, disable_grouping=disable_grouping
420 - )
421 - processed_images_grouped = {}
422 - processed_grids = {}
423 -
424 - for shape, stacked_images in grouped_images.items():
425 - resized_height, resized_width = stacked_images.shape[-2:]
426 -
427 - patches = self.rescale_and_normalize(
428 - stacked_images,
429 - do_rescale,
430 - rescale_factor,
431 - do_normalize,
432 - image_mean,
433 - image_std,
434 - )
435 - if patches.ndim == 4: # (B, C, H, W)
436 - patches = patches.unsqueeze(1) # (B, T=1, C, H, W)
437 -
438 - if patches.shape[1] % temporal_patch_size != 0:
439 - repeats = patches[:, -1:].repeat(
440 - 1,
441 - temporal_patch_size - (patches.shape[1] % temporal_patch_size),
442 - 1,
443 - 1,
444 - 1,
445 - )
446 - patches = torch.cat([patches, repeats], dim=1)
447 -
448 - batch_size, t_len, channel = patches.shape[:3]
449 - grid_t = t_len // temporal_patch_size
450 - grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
451 -
452 - patches = patches.view(
453 - batch_size,
454 - grid_t,
455 - temporal_patch_size,
456 - channel,
457 - grid_h // merge_size,
458 - merge_size,
459 - patch_size,
460 - grid_w // merge_size,
461 - merge_size,
462 - patch_size,
463 - )
464 - # (B, grid_t, gh, gw, mh, mw, C, tp, ph, pw)
465 - patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
466 -
467 - flatten_patches = patches.reshape(
468 - batch_size,
469 - grid_t * grid_h * grid_w,
470 - channel * temporal_patch_size * patch_size * patch_size,
471 - )
472 -
473 - processed_images_grouped[shape] = flatten_patches
474 - processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
475 -
476 - processed_images = reorder_images(
477 - processed_images_grouped, grouped_images_index
478 - )
479 - processed_grids = reorder_images(processed_grids, grouped_images_index)
480 -
481 - pixel_values = torch.cat(processed_images, dim=0)
482 - image_grid_thw = torch.tensor(processed_grids)
483 -
484 - return BatchFeature(
485 - data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw},
486 - tensor_type=return_tensors,
487 - )
488 -
489 - def preprocess(
490 - self, images: ImageInput, **kwargs: Unpack[Glm5NextImageProcessorKwargs]
491 - ) -> BatchFeature:
492 - return super().preprocess(images, **kwargs)
493 -
494 - def get_number_of_image_patches(
495 - self, height: int, width: int, images_kwargs: dict | None = None
496 - ) -> int:
497 - """Number of image patches (pre-merge) for a given (height, width)."""
498 - images_kwargs = images_kwargs or {}
499 - patch_size = images_kwargs.get("patch_size", self.patch_size)
500 - merge_size = images_kwargs.get("merge_size", self.merge_size)
501 - patch_expand_factor = images_kwargs.get(
502 - "patch_expand_factor", self.patch_expand_factor
503 - )
504 - min_pixels, max_pixels = _pixel_budget(
505 - images_kwargs.get("min_image_tokens", self.min_image_tokens),
506 - images_kwargs.get("max_image_tokens", self.max_image_tokens),
507 - patch_size,
508 - merge_size,
509 - self.temporal_patch_size,
510 - )
511 - resized_height, resized_width = smart_resize(
512 - t=self.temporal_patch_size,
513 - h=height,
514 - w=width,
515 - t_factor=self.temporal_patch_size,
516 - h_factor=patch_size * merge_size * patch_expand_factor,
517 - w_factor=patch_size * merge_size * patch_expand_factor,
518 - min_pixels=min_pixels,
519 - max_pixels=max_pixels,
520 - )
521 - grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
522 - return grid_h * grid_w
523 -
524 -
525 -class Glm5NextVideoProcessorKwargs(VideosKwargs, total=False): # type: ignore[call-arg]
526 - fps: list[float] | float
527 - patch_size: int
528 - temporal_patch_size: int
529 - merge_size: int
530 - patch_expand_factor: int
531 - resize_mode: str | None
532 - target_fps: float | None
533 - max_frames: int | None
534 - fps_interval: int | None
535 - max_frame_count_dynamic: int | None
536 - min_image_tokens: int | None
537 - max_image_tokens: int | None
538 -
539 -
540 -class Glm5NextVideoProcessor(BaseVideoProcessor):
541 - """Fast video processor for GLM-5.3-Flash.
542 -
543 - Shares ``smart_resize`` / the pad-mode geometry / the patchify with the
544 - image processor, and adds GLM-5.3-Flash frame sampling
545 - (``glm_sample_frame_indices``: ``fps_interval`` semantics with a
546 - temporal-patch-scaled greedy walk). Defaults mirror the checkpoint's
547 - ``video_processor`` config.
548 - """
549 -
550 - resample = PILImageResampling.BICUBIC
551 - size = {"longest_edge": 1} # unused: budgets come from the token bounds
552 - image_mean = OPENAI_CLIP_MEAN
553 - image_std = OPENAI_CLIP_STD
554 - do_resize = True
555 - do_rescale = True
556 - do_normalize = True
557 - do_convert_rgb = True
558 - do_sample_frames = True
559 - patch_size = 14
560 - temporal_patch_size = 2
561 - patch_expand_factor = 1
562 - merge_size = 2
563 - valid_kwargs = Glm5NextVideoProcessorKwargs
564 - num_frames = 16
565 - fps = 2
566 - fps_interval = 2.0
567 - max_frame_count_dynamic = 2048
568 - resize_mode = "pad"
569 - min_image_tokens = 16
570 - max_image_tokens = 240000
571 - model_input_names = ["pixel_values_videos", "video_grid_thw"]
572 -
573 - def sample_frames(
574 - self,
575 - metadata: VideoMetadata,
576 - fps: int | float | None = None,
577 - **kwargs,
578 - ) -> np.ndarray:
579 - """Sample frame indices with GLM's fps-interval policy.
580 -
581 - ``fps`` / ``target_fps``, ``max_frames`` and ``fps_interval`` /
582 - ``max_frame_count_dynamic`` are the overrides described in
583 - :func:`glm_sample_frame_indices`.
584 - """
585 - if metadata is None or getattr(metadata, "fps", None) is None:
586 - raise ValueError(
587 - "Asked to sample frames per second but no video metadata was "
588 - "provided which is required when sampling in GLM-5.3-Flash. Please "
589 - "pass in `VideoMetadata` object or set `do_sample_frames=False`."
590 - )
591 -
592 - target_fps = fps if fps is not None else kwargs.get("target_fps")
593 - if target_fps is None:
594 - target_fps = self.fps_interval
595 - indices = glm_sample_frame_indices(
596 - metadata.total_num_frames,
597 - metadata.fps,
598 - metadata.duration or 0,
599 - target_fps=target_fps,
600 - max_frame_count=kwargs.get("max_frames") or self.max_frame_count_dynamic,
601 - temporal_patch_size=self.temporal_patch_size,
602 - )
603 - return np.array(indices)
604 -
605 - def _preprocess(
606 - self,
607 - videos: list[torch.Tensor],
608 - do_convert_rgb: bool = True,
609 - do_resize: bool = True,
610 - size: SizeDict | None = None,
611 - resample: "PILImageResampling | int | None" = PILImageResampling.BICUBIC,
612 - do_rescale: bool = True,
613 - rescale_factor: float = 1 / 255.0,
614 - do_normalize: bool = True,
615 - image_mean: float | list[float] | None = None,
616 - image_std: float | list[float] | None = None,
617 - patch_size: int | None = None,
618 - temporal_patch_size: int | None = None,
619 - patch_expand_factor: int | None = None,
620 - merge_size: int | None = None,
621 - resize_mode: str | None = None,
622 - min_image_tokens: int | None = None,
623 - max_image_tokens: int | None = None,
624 - return_tensors: str | TensorType | None = None,
625 - **kwargs,
626 - ) -> BatchFeature:
627 - patch_expand_factor = self.patch_expand_factor
628 - patch_size = patch_size if patch_size is not None else self.patch_size
629 - temporal_patch_size = (
630 - temporal_patch_size
631 - if temporal_patch_size is not None
632 - else self.temporal_patch_size
633 - )
634 - merge_size = merge_size if merge_size is not None else self.merge_size
635 - resize_mode = resize_mode if resize_mode is not None else self.resize_mode
636 - min_pixels, max_pixels = _pixel_budget(
637 - min_image_tokens if min_image_tokens is not None else self.min_image_tokens,
638 - max_image_tokens if max_image_tokens is not None else self.max_image_tokens,
639 - patch_size,
640 - merge_size,
641 - temporal_patch_size,
642 - )
643 - grouped_videos, grouped_videos_index = group_videos_by_shape(videos)
644 - resized_videos_grouped = {}
645 - for shape, stacked_videos in grouped_videos.items():
646 - if do_convert_rgb:
647 - stacked_videos = self.convert_to_rgb(stacked_videos)
648 - b, t_len, c, h, w = stacked_videos.shape
649 - num_frames, height, width = t_len, h, w
650 - if do_resize:
651 - resized_height, resized_width = smart_resize(
652 - t=num_frames,
653 - h=height,
654 - w=width,
655 - t_factor=temporal_patch_size,
656 - h_factor=patch_size * merge_size * patch_expand_factor,
657 - w_factor=patch_size * merge_size * patch_expand_factor,
658 - min_pixels=min_pixels,
659 - max_pixels=max_pixels,
660 - )
661 - stacked_videos = stacked_videos.view(b * t_len, c, h, w)
662 - stacked_videos = _resize_or_pad(
663 - stacked_videos,
664 - target_height=resized_height,
665 - target_width=resized_width,
666 - resize_mode=resize_mode,
667 - resample=resample,
668 - resize=self.resize,
669 - allow_upscale=(num_frames * height * width < min_pixels),
670 - )
671 - stacked_videos = stacked_videos.view(
672 - b, t_len, c, resized_height, resized_width
673 - )
674 - resized_videos_grouped[shape] = stacked_videos
675 - resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)
676 -
677 - grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos)
678 - processed_videos_grouped = {}
679 - processed_grids = {}
680 - for shape, stacked_videos in grouped_videos.items():
681 - resized_height, resized_width = get_image_size(
682 - stacked_videos[0], channel_dim=ChannelDimension.FIRST
683 - )
684 - stacked_videos = self.rescale_and_normalize(
685 - stacked_videos,
686 - do_rescale,
687 - rescale_factor,
688 - do_normalize,
689 - image_mean,
690 - image_std,
691 - )
692 - patches = stacked_videos
693 -
694 - if pad := -patches.shape[1] % temporal_patch_size:
695 - repeats = patches[:, -1:].expand(-1, pad, -1, -1, -1)
696 - patches = torch.cat((patches, repeats), dim=1)
697 - batch_size, grid_t, channel = patches.shape[:3]
698 - grid_t = grid_t // temporal_patch_size
699 - grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
700 -
701 - patches = patches.view(
702 - batch_size,
703 - grid_t,
704 - temporal_patch_size,
705 - channel,
706 - grid_h // merge_size,
707 - merge_size,
708 - patch_size,
709 - grid_w // merge_size,
710 - merge_size,
711 - patch_size,
712 - )
713 - patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
714 - flatten_patches = patches.reshape(
715 - batch_size,
716 - grid_t * grid_h * grid_w,
717 - channel * temporal_patch_size * patch_size * patch_size,
718 - )
719 -
720 - processed_videos_grouped[shape] = flatten_patches
721 - processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
722 -
723 - processed_videos = reorder_videos(
724 - processed_videos_grouped, grouped_videos_index
725 - )
726 - processed_grids = reorder_videos(processed_grids, grouped_videos_index)
727 - pixel_values_videos = torch.cat(processed_videos, dim=0)
728 - video_grid_thw = torch.tensor(processed_grids)
729 - return BatchFeature(
730 - data={
731 - "pixel_values_videos": pixel_values_videos,
732 - "video_grid_thw": video_grid_thw,
733 - },
734 - tensor_type=return_tensors,
735 - )
736 -
737 -
738 -class Glm5NextProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
739 - images_kwargs: Glm5NextImageProcessorKwargs
740 - videos_kwargs: Glm5NextVideoProcessorKwargs
741 - _defaults = {
742 - "text_kwargs": {
743 - "padding": False,
744 - "return_token_type_ids": False,
745 - "return_mm_token_type_ids": False,
746 - },
747 - "videos_kwargs": {"return_metadata": True},
748 - }
749 -
750 -
751 -class Glm5NextProcessor(ProcessorMixin):
752 - """Wraps a GLM-5.3-Flash image processor, video processor and tokenizer.
753 -
754 - Token expansion per image = ``prod(image_grid_thw) // merge_size**2``; video
755 - frames are expanded with ``<|begin_of_image|>...<|end_of_image|>{ts} seconds``
756 - structure (mrope timestamps).
757 - """
758 -
759 - attributes = ["image_processor", "tokenizer", "video_processor"]
760 - image_processor_class = "AutoImageProcessor"
761 - video_processor_class = "AutoVideoProcessor"
762 - tokenizer_class = ("PythonBackend", "TokenizersBackend")
763 -
764 - def __init__(
765 - self,
766 - image_processor=None,
767 - tokenizer=None,
768 - video_processor=None,
769 - chat_template=None,
770 - **kwargs,
771 - ) -> None:
772 - super().__init__(
773 - image_processor, tokenizer, video_processor, chat_template=chat_template
774 - )
775 - self.image_token = (
776 - "<|image|>"
777 - if not hasattr(tokenizer, "image_token")
778 - else tokenizer.image_token
779 - )
780 - self.video_token = (
781 - "<|video|>"
782 - if not hasattr(tokenizer, "video_token")
783 - else tokenizer.video_token
784 - )
785 - self.image_token_id = (
786 - tokenizer.image_token_id
787 - if getattr(tokenizer, "image_token_id", None)
788 - else tokenizer.convert_tokens_to_ids(self.image_token)
789 - )
790 - self.video_token_id = (
791 - tokenizer.video_token_id
792 - if getattr(tokenizer, "video_token_id", None)
793 - else tokenizer.convert_tokens_to_ids(self.video_token)
794 - )
795 -
796 - @classmethod
797 - def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
798 - """Build the processor directly from the checkpoint config.
799 -
800 - GLM-5.3-Flash stores nested image/video configs in
801 - ``processor_config.json`` and declares a custom processor class. This
802 - method reads those configs directly and caps only the video token budget.
803 - """
804 - from transformers import AutoTokenizer
805 -
806 - model_path = pretrained_model_name_or_path
807 - tokenizer = AutoTokenizer.from_pretrained(model_path, **kwargs)
808 -
809 - def _cap_cfg(cfg: dict, *, is_video: bool) -> dict:
810 - # Video keeps a serving token cap (the checkpoint's 240k-token
811 - # budget would starve the KV cache at startup profiling); images
812 - # follow the checkpoint budget verbatim so preprocessing matches
813 - # the HF reference exactly.
814 - if is_video and cfg.get("max_image_tokens") is not None:
815 - cfg["max_image_tokens"] = min(
816 - cfg["max_image_tokens"], _MAX_VIDEO_TOKENS
817 - )
818 - return cfg
819 -
820 - ip_cfg = _cap_cfg(
821 - dict(get_image_processor_config(model_path, **kwargs)), is_video=False
822 - )
823 - image_processor = Glm5NextImageProcessor(
824 - **{k: v for k, v in ip_cfg.items() if k != "image_processor_type"}
825 - )
826 -
827 - processor_config = get_hf_file_to_dict(
828 - "processor_config.json",
829 - model_path,
830 - revision=kwargs.get("revision", "main"),
831 - )
832 - if processor_config is None:
833 - raise ValueError(f"Missing processor_config.json for {model_path}")
834 - vp_cfg = _cap_cfg(dict(processor_config["video_processor"]), is_video=True)
835 - video_processor = Glm5NextVideoProcessor(
836 - **{k: v for k, v in vp_cfg.items() if k != "video_processor_type"}
837 - )
838 -
839 - return cls(
840 - image_processor=image_processor,
841 - tokenizer=tokenizer,
842 - video_processor=video_processor,
843 - )
844 -
845 - def __call__(
846 - self,
847 - images: ImageInput | None = None,
848 - text: TextInput
849 - | PreTokenizedInput
850 - | list[TextInput]
851 - | list[PreTokenizedInput] = None,
852 - videos: VideoInput | None = None,
853 - **kwargs: Unpack[Glm5NextProcessorKwargs],
854 - ) -> BatchFeature:
855 - output_kwargs = self._merge_kwargs(
856 - Glm5NextProcessorKwargs,
857 - tokenizer_init_kwargs=self.tokenizer.init_kwargs,
858 - **kwargs,
859 - )
860 - if images is not None:
861 - image_inputs = self.image_processor(
862 - images=images, **output_kwargs["images_kwargs"]
863 - )
864 - else:
865 - image_inputs = {}
866 -
867 - if videos is not None:
868 - videos_inputs = self.video_processor(
869 - videos=videos, **output_kwargs["videos_kwargs"]
870 - )
871 - if "return_metadata" not in kwargs:
872 - videos_inputs.pop("video_metadata")
873 - else:
874 - videos_inputs = {}
875 -
876 - if not isinstance(text, list):
877 - text = [text]
878 -
879 - # Prompt updates expand the unchanged image/video markers after this call.
880 - return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
881 - return_mm_token_type_ids = output_kwargs["text_kwargs"].pop(
882 - "return_mm_token_type_ids", False
883 - )
884 - text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
885 -
886 - if return_mm_token_type_ids:
887 - array_ids = np.array(text_inputs["input_ids"])
888 - mm_token_type_ids = np.zeros_like(text_inputs["input_ids"])
889 - mm_token_type_ids[array_ids == self.image_token_id] = 1
890 - text_inputs["mm_token_type_ids"] = mm_token_type_ids.tolist()
891 - return BatchFeature(
892 - data={**text_inputs, **image_inputs, **videos_inputs},
893 - tensor_type=return_tensors,
894 - )
895 -
896 - def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, **kwargs):
897 - vision_data = {}
898 - if image_sizes is not None:
899 - images_kwargs = Glm5NextProcessorKwargs._defaults.get("images_kwargs", {})
900 - images_kwargs.update(kwargs)
901 - merge_size = (
902 - images_kwargs.get("merge_size", None) or self.image_processor.merge_size
903 - )
904 -
905 - num_image_patches = [
906 - self.image_processor.get_number_of_image_patches(
907 - *image_size, images_kwargs
908 - )
909 - for image_size in image_sizes
910 - ]
911 - num_image_tokens = [(n // merge_size**2) for n in num_image_patches]
912 - vision_data.update(
913 - {
914 - "num_image_tokens": num_image_tokens,
915 - "num_image_patches": num_image_patches,
916 - }
917 - )
918 -
919 - if video_sizes is not None:
920 - videos_kwargs = Glm5NextProcessorKwargs._defaults.get("videos_kwargs", {})
921 - videos_kwargs.update(kwargs)
922 - num_video_patches = [
923 - self.video_processor.get_number_of_video_patches(
924 - *video_size, videos_kwargs
925 - )
926 - for video_size in video_sizes
927 - ]
928 - num_video_tokens = [(n // merge_size**2) for n in num_video_patches]
929 - vision_data["num_video_tokens"] = num_video_tokens
930 -
931 - return MultiModalData(**vision_data)
932 -
933 - def post_process_image_text_to_text(
934 - self,
935 - generated_outputs,
936 - skip_special_tokens=True,
937 - clean_up_tokenization_spaces=False,
938 - **kwargs,
939 - ):
940 - return self.tokenizer.batch_decode(
941 - generated_outputs,
942 - skip_special_tokens=skip_special_tokens,
943 - clean_up_tokenization_spaces=clean_up_tokenization_spaces,
944 - **kwargs,
945 - )
946 -
947 -
948 -__all__ = [
949 - "Glm5NextImageProcessor",
950 - "Glm5NextVideoProcessor",
951 - "Glm5NextProcessor",
952 - "smart_resize",
953 -]