A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[Bugfix] Fix glm4.1v video_grid_thw tensor shape scheme (#21744)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Jul 28, 2025 at 19:26 UTC
0ae970ed15eb418eccaaef1cc6f7d88cb2eab563
1 file changed
+3
-6
vllm/model_executor/models/glm4_1v.py
+3
-6
@@ -126,7 +126,6 @@ class Glm4vVideoPixelInputs(TensorSchema):
126
- np: Number of patches
127
- ctpp: Number of channels * temporal_patch_size *
128
patch_size * patch_size
129
- - nv: Number of videos
129
- f: Number of frames
130
- g: Grid dimensions (3 for grid_t which is usually 1 for processed
131
video, grid_h, grid_w)
@@ -134,8 +133,7 @@ class Glm4vVideoPixelInputs(TensorSchema):
133
type: Literal["pixel_values_videos"] = "pixel_values_videos"
134
135
pixel_values_videos: Annotated[torch.Tensor, TensorShape("np", "ctpp")]
137
- # video_metadata: Union[list[VideoMetadata], list[dict]]
138
- video_grid_thw: Annotated[torch.Tensor, TensorShape("nv", "f", 3)]
136
+ video_grid_thw: Annotated[torch.Tensor, TensorShape("f", 3)]
137
138
139
class Glm4vVideoEmbeddingInputs(TensorSchema):
@@ -143,14 +141,14 @@ class Glm4vVideoEmbeddingInputs(TensorSchema):
141
Dimensions:
142
- p: Number of video patches across all frames
143
- h: Hidden size (must match language model backbone)
146
- - n: Number of videos
144
+ - f: Number of frames
145
- g: Grid dimensions (3 for grid_t which is usually 1 for processed
146
video, grid_h, grid_w)
147
"""
148
type: Literal["video_embeds"] = "video_embeds"
149
150
video_embeds: Annotated[torch.Tensor, TensorShape("p", "h")]
153
- video_grid_thw: Annotated[torch.Tensor, TensorShape("n", 1, 3)]
151
+ video_grid_thw: Annotated[torch.Tensor, TensorShape("f", 3)]
152
153
154
Glm4vVideoInputs = Union[Glm4vVideoPixelInputs, Glm4vVideoEmbeddingInputs]
@@ -1348,7 +1346,6 @@ class Glm4vForConditionalGeneration(nn.Module, SupportsMultiModal,
1346
1347
return Glm4vVideoPixelInputs(
1348
type="pixel_values_videos",
1351
- # video_metadata=video_metadata,
1349
pixel_values_videos=pixel_values_videos,
1350
video_grid_thw=video_grid_thw,
1351
)