[Bugfix][GLM-5.3-Flash] Take video placeholder timestamps from the pixel path's frame sampler (#55647)
Signed-off-by: NNNtrance <232268405+NNNtrance@users.noreply.github.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: NNNtrance <232268405+NNNtrance@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
burak committed
Sep 29, 2026 at 09:45 UTC
65ed0a2642d2fb1d897408ec39f64b1cb74f0369
2 files changed
+229
tests/models/multimodal/processing/test_glm5next.py
new
+170
@@ -0,0 +1,170 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Video placeholder accounting for GLM-5.3-Flash.
4
+
5
+``Glm4vProcessingInfo._construct_video_placeholder`` emits one frame of
6
+placeholders per timestamp, so the timestamps returned by
7
+``_get_video_second_idx_glm46v`` decide how many placeholders the prompt gets
8
+while ``video_grid_thw`` decides how many rows the vision tower produces. If
9
+the two disagree they collide in ``_merge_multimodal_embeddings``, which raises
10
+inside a worker and takes the engine down with it.
11
+
12
+The checks below are arithmetic: no weights, no GPU.
13
+"""
14
+
15
+import pytest
16
+
17
+from vllm.multimodal import MULTIMODAL_REGISTRY
18
+from vllm.transformers_utils.processors.glm5next import (
19
+ Glm5NextVideoProcessor,
20
+ _pixel_budget,
21
+ glm_sample_frame_indices,
22
+ smart_resize,
23
+)
24
+
25
+from ...utils import build_model_context
26
+
27
+
28
+@pytest.fixture(scope="module")
29
+def processor():
30
+ ctx = build_model_context(
31
+ "zai-org/GLM-5.3-Flash",
32
+ limit_mm_per_prompt={"video": 1},
33
+ )
34
+ return MULTIMODAL_REGISTRY.create_processor(
35
+ ctx.model_config,
36
+ tokenizer=ctx.tokenizer,
37
+ )
38
+
39
+
40
+def _pixel_path_grid(
41
+ video_processor: Glm5NextVideoProcessor,
42
+ num_frames: int,
43
+ height: int,
44
+ width: int,
45
+) -> tuple[int, int, int]:
46
+ """The ``video_grid_thw`` ``Glm5NextVideoProcessor._preprocess`` builds."""
47
+ min_pixels, max_pixels = _pixel_budget(
48
+ video_processor.min_image_tokens,
49
+ video_processor.max_image_tokens,
50
+ video_processor.patch_size,
51
+ video_processor.merge_size,
52
+ video_processor.temporal_patch_size,
53
+ )
54
+ factor = (
55
+ video_processor.patch_size
56
+ * video_processor.merge_size
57
+ * video_processor.patch_expand_factor
58
+ )
59
+ resized_height, resized_width = smart_resize(
60
+ t=num_frames,
61
+ h=height,
62
+ w=width,
63
+ t_factor=video_processor.temporal_patch_size,
64
+ h_factor=factor,
65
+ w_factor=factor,
66
+ min_pixels=min_pixels,
67
+ max_pixels=max_pixels,
68
+ )
69
+ padded_frames = num_frames + (-num_frames % video_processor.temporal_patch_size)
70
+ return (
71
+ padded_frames // video_processor.temporal_patch_size,
72
+ resized_height // video_processor.patch_size,
73
+ resized_width // video_processor.patch_size,
74
+ )
75
+
76
+
77
+@pytest.mark.parametrize(
78
+ ("total_num_frames", "fps", "duration", "height", "width", "expected_grid"),
79
+ [
80
+ # 4 s at 8 fps: the GLM-4.6V sampler asks for 3x as many timestamps.
81
+ (32, 8.0, 4.0, 480, 640, (4, 36, 46)),
82
+ # 1080p, 20 s: same factor of 3 at a full-size canvas.
83
+ (600, 30.0, 20.0, 1080, 1920, (20, 58, 102)),
84
+ # 1080p, 60 s: the one duration window where the two samplers agree
85
+ # anyway -- a regression guard, the count must not move.
86
+ (1800, 30.0, 60.0, 1080, 1920, (60, 34, 58)),
87
+ # Past 300 s the GLM-4.6V sampler asks for half as many instead.
88
+ (9030, 30.0, 301.0, 720, 1280, (301, 14, 26)),
89
+ ],
90
+)
91
+def test_video_placeholders_match_encoder_rows(
92
+ processor,
93
+ total_num_frames: int,
94
+ fps: float,
95
+ duration: float,
96
+ height: int,
97
+ width: int,
98
+ expected_grid: tuple[int, int, int],
99
+):
100
+ info = processor.info
101
+ video_processor = info.get_video_processor()
102
+
103
+ frame_indices = glm_sample_frame_indices(
104
+ total_num_frames,
105
+ fps,
106
+ duration,
107
+ target_fps=video_processor.fps_interval,
108
+ max_frame_count=video_processor.max_frame_count_dynamic,
109
+ temporal_patch_size=video_processor.temporal_patch_size,
110
+ )
111
+ grid_t, grid_h, grid_w = _pixel_path_grid(
112
+ video_processor, len(frame_indices), height, width
113
+ )
114
+ assert (grid_t, grid_h, grid_w) == expected_grid
115
+
116
+ timestamps = info._get_video_second_idx_glm46v(
117
+ {
118
+ "total_num_frames": total_num_frames,
119
+ "fps": fps,
120
+ "duration": duration,
121
+ "do_sample_frames": True,
122
+ },
123
+ total_num_frames,
124
+ )
125
+
126
+ merge_length = video_processor.merge_size**2
127
+ tokens_per_frame = grid_h * grid_w // merge_length
128
+ encoder_rows = grid_t * grid_h * grid_w // merge_length
129
+
130
+ assert len(timestamps) == grid_t
131
+ assert len(timestamps) * tokens_per_frame == encoder_rows
132
+ assert timestamps == sorted(timestamps)
133
+ assert timestamps[0] == 0
134
+ assert timestamps[-1] <= duration
135
+
136
+
137
+def test_video_placeholders_match_encoder_rows_when_presampled(processor):
138
+ """The loader may pre-sample and hand the frames over as they are."""
139
+ info = processor.info
140
+ video_processor = info.get_video_processor()
141
+
142
+ num_frames = 32
143
+ grid_t, _, _ = _pixel_path_grid(video_processor, num_frames, 480, 640)
144
+
145
+ timestamps = info._get_video_second_idx_glm46v(
146
+ {
147
+ "total_num_frames": 256,
148
+ "fps": 8.0,
149
+ "duration": 32.0,
150
+ "do_sample_frames": False,
151
+ "frames_indices": list(range(0, 256, 256 // num_frames)),
152
+ },
153
+ num_frames,
154
+ )
155
+
156
+ assert len(timestamps) == grid_t
157
+
158
+
159
+def test_video_shorter_than_one_sampling_interval_is_rejected(processor):
160
+ """A clip the sampler cannot pick a single frame from is a bad request."""
161
+ with pytest.raises(ValueError, match="selected no frames"):
162
+ processor.info._get_video_second_idx_glm46v(
163
+ {
164
+ "total_num_frames": 2,
165
+ "fps": 8.0,
166
+ "duration": 0.25,
167
+ "do_sample_frames": True,
168
+ },
169
+ 2,
170
+ )
vllm/models/glm5next/common/multimodal.py
+59
@@ -4,6 +4,7 @@
4
5
from collections.abc import Mapping
6
from functools import cached_property, partial
7
+from typing import Any
8
9
import numpy as np
10
import torch
@@ -42,6 +43,7 @@ from vllm.model_executor.models.vision import (
43
)
44
from vllm.models.common.ops import fused_q_kv_rmsnorm
45
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
46
+from vllm.transformers_utils.processors.glm5next import glm_sample_frame_indices
47
from vllm.v1.attention.backends.registry import AttentionBackendEnum
48
49
@@ -656,6 +658,63 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
658
return int(override)
659
return self._processor_pixel_budget(self.get_hf_processor().video_processor)[1]
660
661
+ def _get_video_second_idx_glm46v(
662
+ self, metadata: dict[str, Any], total_frames: int
663
+ ) -> list[int]:
664
+ """Timestamps for the video placeholder, from the pixel path's sampler.
665
+
666
+ The inherited GLM-4.6V timestamp sampling uses different constants
667
+ than this checkpoint's ``Glm5NextVideoProcessor`` pixel sampling, so
668
+ placeholder count and ``video_grid_thw`` disagree outside 30-300s
669
+ durations. Sampling with the pixel path's own policy makes one
670
+ timestamp per temporal patch == ``grid_t``.
671
+ """
672
+ video_processor = self.get_video_processor()
673
+ temporal_patch_size = int(
674
+ getattr(video_processor, "temporal_patch_size", 1) or 1
675
+ )
676
+ video_fps = float(metadata["fps"])
677
+
678
+ if not metadata.get("do_sample_frames", True):
679
+ # The loader already picked the frames and the processor keeps them
680
+ # as they are, so the frame count is what was handed in and
681
+ # ``frames_indices`` only supplies the timestamp values.
682
+ frame_indices = [int(idx) for idx in metadata["frames_indices"]]
683
+ num_frames = int(total_frames)
684
+ else:
685
+ frame_indices = [
686
+ int(idx)
687
+ for idx in glm_sample_frame_indices(
688
+ int(metadata.get("total_num_frames", total_frames)),
689
+ video_fps,
690
+ float(metadata.get("duration") or 0),
691
+ target_fps=video_processor.fps_interval,
692
+ max_frame_count=video_processor.max_frame_count_dynamic,
693
+ temporal_patch_size=temporal_patch_size,
694
+ )
695
+ ]
696
+ num_frames = len(frame_indices)
697
+
698
+ if not frame_indices or num_frames <= 0:
699
+ # ``duration * fps_interval < 1`` selects no frames at all; the
700
+ # pixel path raises an opaque IndexError on the same input a moment
701
+ # later, so say what happened while there is still a request.
702
+ raise ValueError(
703
+ "The GLM-5-Next frame sampler selected no frames for this "
704
+ f"video (fps={video_fps}, duration={metadata.get('duration')!r},"
705
+ f" total_num_frames={metadata.get('total_num_frames')!r}). "
706
+ "Clips shorter than one frame sampling interval cannot be "
707
+ "processed."
708
+ )
709
+
710
+ num_padded_frames = num_frames + (-num_frames % temporal_patch_size)
711
+ grid_t = max(num_padded_frames // temporal_patch_size, 1)
712
+ last = len(frame_indices) - 1
713
+ return [
714
+ int(frame_indices[min(i * temporal_patch_size, last)] / video_fps)
715
+ for i in range(grid_t)
716
+ ]
717
+
718
def _get_vision_info(
719
self,
720
*,