A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 22:15 UTC
GLM-V video segmentation solution adjustment (#28941)
Signed-off-by: zRzRzRzRzRzRzR <2448370773@qq.com>
Yuxuan Zhang committed
Nov 20, 2025 at 01:32 UTC
0c80efd94fb8c17cfc7d1bcb9cdb65f154340994
1 file changed
+90
-4
vllm/model_executor/models/glm4_1v.py
+90
-4
@@ -37,7 +37,7 @@ import torch
37
import torch.nn as nn
38
import torch.nn.functional as F
39
from einops import rearrange
40
-from transformers import BatchFeature
40
+from transformers import BatchFeature, Glm4vProcessor
41
from transformers.models.glm4v.configuration_glm4v import Glm4vVisionConfig
42
from transformers.models.glm4v.image_processing_glm4v import (
43
Glm4vImageProcessor,
@@ -1028,7 +1028,7 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
1028
1029
return max(max_frames_per_video, 1)
1030
1031
- def _get_video_second_idx(
1031
+ def _get_video_second_idx_glm4v(
1032
self, metadata: dict[str, Any], total_frames: int
1033
) -> list[int]:
1034
video_processor = self.get_video_processor()
@@ -1079,6 +1079,83 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
1079
selected_timestamps.append(timestamps_list[idx])
1080
return selected_timestamps
1081
1082
+ def _get_video_second_idx_glm46v(
1083
+ self, metadata: dict[str, Any], total_frames: int
1084
+ ) -> list[int]:
1085
+ video_processor = self.get_video_processor()
1086
+
1087
+ video_fps = metadata["fps"]
1088
+ meta_frames = metadata.get("total_num_frames", total_frames)
1089
+ max_frame_idx = meta_frames - 1
1090
+ duration = metadata.get("duration", round(max_frame_idx / video_fps) + 1)
1091
+
1092
+ do_sample_frames = metadata.get("do_sample_frames", True)
1093
+ if not do_sample_frames:
1094
+ frame_indices = metadata["frames_indices"]
1095
+ else:
1096
+ DYNAMIC_FPS_THRES = {30: 3, 300: 1, 2400: 0.5}
1097
+ MAX_FRAME_COUNT_DYNAMIC = 640
1098
+ MAX_DURATION = 2400
1099
+
1100
+ effective_duration = min(duration, MAX_DURATION)
1101
+ if effective_duration <= 30:
1102
+ target_fps = DYNAMIC_FPS_THRES[30]
1103
+ elif effective_duration <= 300:
1104
+ target_fps = DYNAMIC_FPS_THRES[300]
1105
+ else:
1106
+ target_fps = DYNAMIC_FPS_THRES[2400]
1107
+
1108
+ temporal_patch_size = getattr(video_processor, "temporal_patch_size", 1)
1109
+ extract_t = int(effective_duration * target_fps * temporal_patch_size)
1110
+ extract_t = min(extract_t, MAX_FRAME_COUNT_DYNAMIC)
1111
+
1112
+ duration_per_frame = 1 / video_fps
1113
+ timestamps = [i * duration_per_frame for i in range(meta_frames)]
1114
+ max_second = int(duration)
1115
+
1116
+ if meta_frames < extract_t:
1117
+ frame_indices = np.linspace(
1118
+ 0, meta_frames - 1, extract_t, dtype=int
1119
+ ).tolist()
1120
+ else:
1121
+ frame_indices = []
1122
+ current_second = 0.0
1123
+ inv_fps = 1 / (temporal_patch_size * target_fps)
1124
+ for frame_index in range(meta_frames):
1125
+ if timestamps[frame_index] >= current_second:
1126
+ current_second += inv_fps
1127
+ frame_indices.append(frame_index)
1128
+ if current_second >= max_second:
1129
+ break
1130
+
1131
+ if len(frame_indices) < extract_t:
1132
+ if len(frame_indices) == 0:
1133
+ start, end = 0, max(meta_frames - 1, 0)
1134
+ else:
1135
+ start, end = frame_indices[0], frame_indices[-1]
1136
+ frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()
1137
+ elif len(frame_indices) > extract_t:
1138
+ frame_indices = np.linspace(
1139
+ 0, meta_frames - 1, extract_t, dtype=int
1140
+ ).tolist()
1141
+
1142
+ seen, uniq = set(), []
1143
+ for idx in frame_indices:
1144
+ if idx not in seen:
1145
+ seen.add(idx)
1146
+ uniq.append(idx)
1147
+
1148
+ if len(uniq) & 1:
1149
+ uniq.append(uniq[-1])
1150
+
1151
+ frame_indices = uniq
1152
+ full_second_idxs = [int(idx / video_fps) for idx in frame_indices]
1153
+ timestamps_list = full_second_idxs[::2]
1154
+ selected_timestamps = []
1155
+ for idx in range(len(timestamps_list)):
1156
+ selected_timestamps.append(timestamps_list[idx])
1157
+ return selected_timestamps
1158
+
1159
def _construct_video_placeholder(
1160
self,
1161
video_array: np.ndarray,
@@ -1097,9 +1174,18 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
1174
merge_length = image_processor.merge_size**2
1175
1176
assert isinstance(grid_thw, torch.Tensor)
1100
- timestamps = self._get_video_second_idx(metadata, len(video_array))
1177
+ timestamps = (
1178
+ self._get_video_second_idx_glm4v(metadata, len(video_array))
1179
+ if isinstance(hf_processor, Glm4vProcessor)
1180
+ else self._get_video_second_idx_glm46v(metadata, len(video_array))
1181
+ )
1182
+
1183
+ timestamp_format = (
1184
+ "{}" if isinstance(hf_processor, Glm4vProcessor) else "{:.1f} seconds"
1185
+ )
1186
frames_idx_token = [
1102
- tokenizer.encode(str(i), add_special_tokens=False) for i in timestamps
1187
+ tokenizer.encode(timestamp_format.format(i), add_special_tokens=False)
1188
+ for i in timestamps
1189
]
1190
T, H, W = grid_thw
1191
num_tokens_per_frame = int(H * W) // merge_length