@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 212deff2ec

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 2, 12:15 UTC

[feat] add GlmgaProcessor specific logits in `glm4_1v.py` (#43575)

Signed-off-by: JaredforReal <w13431838023@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <Isotr0py@outlook.com>

Jared Wen committed May 29, 2026 at 10:56 UTC 212deff2ec77a6154c69a5fcc5b51a702b3d6c91

3 files changed +346 -33

tests/models/registry.py

+4 -1

@@ -935,7 +935,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {

935 trust_remote_code=True,

936 hf_overrides={"architectures": ["GLM4VForCausalLM"]},

937 ),

938 - "Glm4vForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.1V-9B-Thinking"),

938 + "Glm4vForConditionalGeneration": _HfExamplesInfo(

939 + "zai-org/GLM-4.1V-9B-Thinking",

940 + extras={"4.6V": "zai-org/GLM-4.6V-Flash"},

941 + ),

942 "Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V"),

943 "GlmOcrForConditionalGeneration": _HfExamplesInfo(

944 "zai-org/GLM-OCR",

vllm/model_executor/models/glm4_1v.py

+241 -32

@@ -36,7 +36,9 @@ import numpy as np

36 import torch

37 import torch.nn as nn

38 import torch.nn.functional as F

39 +import transformers

40 from einops import rearrange

41 +from packaging.version import Version

42 from transformers import BatchFeature, Glm4vProcessor

43 from transformers.models.glm4v.configuration_glm4v import (

44 Glm4vTextConfig,

@@ -122,6 +124,15 @@ logger = init_logger(__name__)

124 # For profile run

125 _MAX_FRAMES_PER_VIDEO = 600

126

127 +TRANSFORMERS_WITH_GA = Version(transformers.__version__) >= Version("5.10.0.dev0")

128 +

129 +

130 +def _to_video_metadata(metadata: Mapping[str, Any]) -> VideoMetadata:

131 + return VideoMetadata(

132 + **{k: metadata[k] for k in metadata if k != "do_sample_frames"}

133 + )

134 +

135 +

136 # === Vision Inputs === #

137

138

@@ -832,6 +843,46 @@ class Glm4vProcessingInfo(BaseProcessingInfo):

843 def get_video_processor(self, **kwargs: object) -> Glm4vVideoProcessor:

844 return self.get_hf_processor(**kwargs).video_processor

845

846 + def get_mm_max_tokens_per_item(

847 + self,

848 + seq_len: int,

849 + mm_counts: Mapping[str, int],

850 + ) -> Mapping[str, int] | None:

851 + processor = self.get_hf_processor()

852 + if isinstance(processor, Glm4vProcessor):

853 + return None

854 +

855 + result: dict[str, int] = {}

856 +

857 + if mm_counts.get("image", 0) > 0:

858 + result["image"] = self.get_max_image_tokens()

859 +

860 + if mm_counts.get("video", 0) > 0:

861 + video_processor = self.get_video_processor()

862 + max_pixels = video_processor.size["longest_edge"]

863 +

864 + vision_config = self.get_hf_config().vision_config

865 + temporal_patch_size = vision_config.temporal_patch_size

866 + patch_size = vision_config.patch_size

867 + merge_size = vision_config.spatial_merge_size

868 +

869 + max_vision_tokens = max_pixels // (

870 + temporal_patch_size * patch_size**2 * merge_size**2

871 + )

872 +

873 + # GLMGA supports up to 640 frames (max_frames).

874 + max_grid_t = 640 // temporal_patch_size

875 +

876 + tokenizer = self.get_tokenizer()

877 + max_ts_tokens = max(

878 + len(tokenizer.encode(f"{t:.1f} seconds", add_special_tokens=False))

879 + for t in range(min(max_grid_t, 300))

880 + )

881 +

882 + result["video"] = max_vision_tokens + max_grid_t * (2 + max_ts_tokens) + 2

883 +

884 + return result

885 +

886 def get_data_parser(self):

887 return MultiModalDataParser(

888 video_needs_metadata=True,

@@ -1104,12 +1155,83 @@ class Glm4vProcessingInfo(BaseProcessingInfo):

1155 selected_timestamps.append(timestamps_list[idx])

1156 return selected_timestamps

1157

1158 + def _is_glmga_model(self, processor: object) -> bool:

1159 + """Detect GLMGA variant via its Glmga sub-processors."""

1160 + for attr in ("image_processor", "video_processor"):

1161 + sub = getattr(processor, attr, None)

1162 + if sub and "Glmga" in type(sub).__name__:

1163 + return True

1164 + return False

1165 +

1166 + def _get_video_second_idx_glmga(

1167 + self, metadata: dict[str, Any], total_frames: int

1168 + ) -> list[int]:

1169 + """Fixed fps=2 frame selection matching GlmgaVideoProcessor.sample_frames."""

1170 + video_processor = self.get_video_processor()

1171 +

1172 + video_fps = metadata["fps"]

1173 + meta_frames = metadata.get("total_num_frames", total_frames)

1174 + max_frame_idx = meta_frames - 1

1175 + duration = metadata.get("duration", round(max_frame_idx / video_fps) + 1)

1176 +

1177 + do_sample_frames = metadata.get("do_sample_frames", True)

1178 + if not do_sample_frames:

1179 + frame_indices = metadata["frames_indices"]

1180 + else:

1181 + target_fps = 2

1182 + max_frames = getattr(video_processor, "max_frames", 640)

1183 + extract_t = int(duration * target_fps)

1184 + extract_t = min(extract_t, max_frames)

1185 +

1186 + duration_per_frame = 1 / video_fps

1187 + timestamps = [i * duration_per_frame for i in range(meta_frames)]

1188 +

1189 + if meta_frames < extract_t:

1190 + frame_indices = [

1191 + math.floor(i * meta_frames / extract_t) for i in range(extract_t)

1192 + ]

1193 + else:

1194 + frame_indices = []

1195 + current_second = 0.0

1196 + inv_fps = 1 / target_fps

1197 + for frame_index in range(meta_frames):

1198 + if timestamps[frame_index] >= current_second:

1199 + current_second += inv_fps

1200 + frame_indices.append(frame_index)

1201 + if current_second >= duration - inv_fps:

1202 + break

1203 +

1204 + if len(frame_indices) < extract_t:

1205 + if len(frame_indices) == 0:

1206 + start, end = 0, max(meta_frames - 1, 0)

1207 + else:

1208 + start, end = frame_indices[0], frame_indices[-1]

1209 + frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()

1210 + elif len(frame_indices) > extract_t:

1211 + frame_indices = np.linspace(

1212 + 0, meta_frames - 1, extract_t, dtype=int

1213 + ).tolist()

1214 +

1215 + seen, uniq = set(), []

1216 + for idx in frame_indices:

1217 + if idx not in seen:

1218 + seen.add(idx)

1219 + uniq.append(idx)

1220 +

1221 + if len(uniq) & 1:

1222 + uniq.append(uniq[-1])

1223 +

1224 + frame_indices = uniq

1225 + full_second_idxs = [int(idx / video_fps) for idx in frame_indices]

1226 + timestamps_list = full_second_idxs[::2]

1227 + return list(timestamps_list)

1228 +

1229 def _construct_video_placeholder(

1230 self,

1231 video_array: np.ndarray,

1232 metadata: dict[str, Any],

1233 grid_thw: torch.Tensor,

1112 - ) -> str:

1234 + ) -> list[int]:

1235 hf_processor = self.get_hf_processor()

1236 tokenizer = self.get_tokenizer()

1237 image_processor = hf_processor.image_processor

@@ -1122,11 +1244,13 @@ class Glm4vProcessingInfo(BaseProcessingInfo):

1244 merge_length = image_processor.merge_size**2

1245

1246 assert isinstance(grid_thw, torch.Tensor)

1125 - timestamps = (

1126 - self._get_video_second_idx_glm4v(metadata, len(video_array))

1127 - if isinstance(hf_processor, Glm4vProcessor)

1128 - else self._get_video_second_idx_glm46v(metadata, len(video_array))

1129 - )

1247 +

1248 + if isinstance(hf_processor, Glm4vProcessor):

1249 + timestamps = self._get_video_second_idx_glm4v(metadata, len(video_array))

1250 + elif self._is_glmga_model(hf_processor):

1251 + timestamps = self._get_video_second_idx_glmga(metadata, len(video_array))

1252 + else:

1253 + timestamps = self._get_video_second_idx_glm46v(metadata, len(video_array))

1254

1255 timestamp_format = (

1256 "{}" if isinstance(hf_processor, Glm4vProcessor) else "{:.1f} seconds"

@@ -1139,9 +1263,16 @@ class Glm4vProcessingInfo(BaseProcessingInfo):

1263 num_tokens_per_frame = int(H * W) // merge_length

1264 placeholder = []

1265 placeholder.append(bov_token_id)

1266 + # Glm46VProcessor uses image_token_id for video frame embeddings;

1267 + # Glm4vProcessor uses video_token_id.

1268 + frame_embed_token_id = (

1269 + hf_processor.video_token_id

1270 + if isinstance(hf_processor, Glm4vProcessor) or not TRANSFORMERS_WITH_GA

1271 + else hf_processor.image_token_id

1272 + )

1273 for frame_idx in frames_idx_token:

1274 placeholder.append(boi_token_id)

1144 - placeholder.extend([hf_processor.video_token_id] * num_tokens_per_frame)

1275 + placeholder.extend([frame_embed_token_id] * num_tokens_per_frame)

1276 placeholder.append(eoi_token_id)

1277 placeholder.extend(frame_idx)

1278 placeholder.append(eov_token_id)

@@ -1241,6 +1372,47 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):

1372

1373

1374 class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):

1375 + @staticmethod

1376 + def _get_direct_path_inputs(

1377 + mm_data: Mapping[str, object],

1378 + mm_kwargs: Mapping[str, object],

1379 + ) -> tuple[Mapping[str, object], Mapping[str, object]]:

1380 + prepared_data = dict(mm_data)

1381 + prepared_kwargs = dict(mm_kwargs)

1382 +

1383 + videos = prepared_data.get("videos")

1384 + if not (isinstance(videos, list) and len(videos) > 0):

1385 + return prepared_data, prepared_kwargs

1386 +

1387 + hf_videos = []

1388 + hf_video_metadata = []

1389 + for item in videos:

1390 + if isinstance(item, tuple) and len(item) == 2:

1391 + video_array, metadata = item

1392 + hf_videos.append(video_array)

1393 + if isinstance(metadata, VideoMetadata):

1394 + hf_video_metadata.append(metadata)

1395 + elif isinstance(metadata, Mapping):

1396 + hf_video_metadata.append(_to_video_metadata(metadata))

1397 + if "do_sample_frames" in metadata:

1398 + prepared_kwargs["do_sample_frames"] = metadata[

1399 + "do_sample_frames"

1400 + ]

1401 + elif metadata is not None:

1402 + raise TypeError(

1403 + "Video metadata must be a mapping or VideoMetadata, "

1404 + f"got {type(metadata)}"

1405 + )

1406 + else:

1407 + hf_videos.append(item)

1408 +

1409 + prepared_data["videos"] = hf_videos

1410 + if hf_video_metadata:

1411 + prepared_data["video_metadata"] = hf_video_metadata

1412 + prepared_kwargs["return_metadata"] = True

1413 +

1414 + return prepared_data, prepared_kwargs

1415 +

1416 def _call_hf_processor(

1417 self,

1418 prompt: str,

@@ -1249,11 +1421,32 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):

1421 tok_kwargs: Mapping[str, object],

1422 ) -> BatchFeature:

1423 mm_data = dict(mm_data)

1424 + if not mm_data:

1425 + tokenizer = self.info.get_tokenizer()

1426 + prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)

1427 + return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")

1428 +

1429 processor = self.info.get_hf_processor(**mm_kwargs)

1430

1254 - # GLM-4.1V use `image_token_id` as video placeholder, we need to

1255 - # replace it with `video_token_id` for video processing. So we

1256 - # separate video processing from image processing.

1431 + # Glm46VProcessor and GLMGA handle image/video placeholders together

1432 + # via the direct path. Only Glm4vProcessor (GLM-4.1V) needs the

1433 + # split-video path because it uses image_token_id as the video

1434 + # placeholder. The direct path requires transformers >= 5.5.0

1435 + # (Glm46VProcessor / GlmgaVideoProcessor support).

1436 + use_direct_path = (

1437 + not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA

1438 + )

1439 + if use_direct_path:

1440 + prepared_data, prepared_kwargs = self._get_direct_path_inputs(

1441 + mm_data, mm_kwargs

1442 + )

1443 + return super()._call_hf_processor(

1444 + prompt=prompt,

1445 + mm_data=prepared_data,

1446 + mm_kwargs=prepared_kwargs,

1447 + tok_kwargs=tok_kwargs,

1448 + )

1449 +

1450 if (

1451 "videos" in mm_data

1452 and isinstance(mm_data["videos"], list)

@@ -1272,19 +1465,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):

1465

1466 video_mm_data = dict()

1467 video_mm_data["videos"] = [[video_array]]

1275 -

1276 - unuse_metadata = ["do_sample_frames"]

1277 - video_mm_data["video_metadata"] = [

1278 - [

1279 - VideoMetadata(

1280 - **{

1281 - k: metadata[k]

1282 - for k in metadata

1283 - if k not in unuse_metadata

1284 - }

1285 - )

1286 - ]

1287 - ]

1468 + video_mm_data["video_metadata"] = [[_to_video_metadata(metadata)]]

1469

1470 video_outputs = super()._call_hf_processor(

1471 prompt="<|begin_of_video|><|video|><|end_of_video|>",

@@ -1366,6 +1547,22 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):

1547 embed_token_id=hf_processor.video_token_id,

1548 )

1549

1550 + def get_video_replacement_glm46v(item_idx: int):

1551 + out_item = out_mm_kwargs["video"][item_idx]

1552 + grid_thw = out_item["video_grid_thw"].data

1553 + assert isinstance(grid_thw, torch.Tensor)

1554 +

1555 + video, metadata = mm_items["video"][item_idx]

1556 + placeholder = self.info._construct_video_placeholder(

1557 + video, metadata, grid_thw

1558 + )

1559 + return PromptUpdateDetails.select_token_id(

1560 + placeholder,

1561 + embed_token_id=hf_processor.image_token_id,

1562 + )

1563 +

1564 + is_glm46v = not isinstance(hf_processor, Glm4vProcessor)

1565 +

1566 return [

1567 PromptReplacement(

1568 modality="image",

@@ -1375,7 +1572,11 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):

1572 PromptReplacement(

1573 modality="video",

1574 target="<|begin_of_video|><|video|><|end_of_video|>",

1378 - replacement=get_video_replacement_glm4v,

1575 + replacement=(

1576 + get_video_replacement_glm46v

1577 + if is_glm46v and TRANSFORMERS_WITH_GA

1578 + else get_video_replacement_glm4v

1579 + ),

1580 ),

1581 ]

1582

@@ -1436,7 +1637,7 @@ class Glm4vForConditionalGeneration(

1637 prefix=maybe_prefix(prefix, "visual"),

1638 )

1639

1439 - if config.model_type in ("glm4v", "glm_ocr"):

1640 + if config.model_type in ("glm4v", "glm_ocr", "glmga"):

1641 architectures = ["Glm4ForCausalLM"]

1642 elif config.model_type == "glm4v_moe":

1643 architectures = ["Glm4MoeForCausalLM"]

@@ -1600,19 +1801,27 @@ class Glm4vForConditionalGeneration(

1801 hf_config = self.config

1802 spatial_merge_size = hf_config.vision_config.spatial_merge_size

1803 for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):

1603 - offset = mm_feature.mm_position.offset

1804 + embed_ranges = mm_feature.mm_position.extract_embeds_range()

1805 if mm_feature.modality == "image":

1806 t, h, w = mm_feature.data["image_grid_thw"].data.tolist()

1807 assert t == 1, f"Image must have 1 frame, got {t}"

1808 + assert len(embed_ranges) == 1

1809 + offset, end = embed_ranges[0]

1810 + assert end - offset + 1 == h * w // spatial_merge_size**2

1811 yield offset, t, h // spatial_merge_size, w // spatial_merge_size

1812 elif mm_feature.modality == "video":

1813 t, h, w = mm_feature.data["video_grid_thw"].data.tolist()

1610 - yield (

1611 - offset,

1612 - t,

1613 - h // spatial_merge_size,

1614 - w // spatial_merge_size,

1615 - )

1814 + llm_grid_h = h // spatial_merge_size

1815 + llm_grid_w = w // spatial_merge_size

1816 + num_tokens_per_frame = llm_grid_h * llm_grid_w

1817 +

1818 + if len(embed_ranges) == t:

1819 + for offset, end in embed_ranges:

1820 + assert end - offset + 1 == num_tokens_per_frame

1821 + yield offset, 1, llm_grid_h, llm_grid_w

1822 + else:

1823 + offset = mm_feature.mm_position.offset

1824 + yield offset, t, llm_grid_h, llm_grid_w

1825 else:

1826 raise ValueError(f"Unsupported modality: {mm_feature.modality}")

1827

vllm/multimodal/video.py

+101

@@ -639,6 +639,107 @@ class DynamicVideoBackend(VideoBackend):

639 )

640

641

642 +@VIDEO_LOADER_REGISTRY.register("glm4_6v")

643 +class GLM4_6VVideoBackend(VideoBackend):

644 + @classmethod

645 + def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata:

646 + # Estimate duration from frame count and fps when the container

647 + # does not report it (common for WebM/streaming inputs).

648 + if source.duration:

649 + return source

650 + if source.original_fps > 0:

651 + max_frame_idx = source.total_frames_num - 1

652 + duration = round(max_frame_idx / source.original_fps) + 1

653 + else:

654 + duration = 0

655 + return VideoSourceMetadata(

656 + source.total_frames_num, source.original_fps, duration

657 + )

658 +

659 + @classmethod

660 + def compute_frames_index_to_sample(

661 + cls,

662 + source: VideoSourceMetadata,

663 + target: VideoTargetMetadata,

664 + **kwargs,

665 + ) -> list[int]:

666 + total_frames_num = source.total_frames_num

667 + duration = source.duration

668 + original_fps = source.original_fps

669 + target_fps = target.fps

670 + max_frame_idx = source.total_frames_num - 1

671 + max_frames = kwargs.get("max_frames", 640)

672 +

673 + duration = duration or round(max_frame_idx / original_fps) + 1

674 +

675 + extract_t = int(duration * target_fps)

676 + extract_t = min(extract_t, max_frames)

677 +

678 + duration_per_frame = 1 / original_fps

679 + timestamps = [i * duration_per_frame for i in range(total_frames_num)]

680 +

681 + if total_frames_num < extract_t:

682 + frame_indices = [

683 + math.floor(i * total_frames_num / extract_t) for i in range(extract_t)

684 + ]

685 + else:

686 + frame_indices = []

687 + current_second = 0.0

688 + inv_fps = 1 / target_fps

689 + for frame_index in range(total_frames_num):

690 + if timestamps[frame_index] >= current_second:

691 + current_second += inv_fps

692 + frame_indices.append(frame_index)

693 + if current_second >= duration - inv_fps:

694 + break

695 +

696 + if len(frame_indices) < extract_t:

697 + if len(frame_indices) == 0:

698 + start, end = 0, max(total_frames_num - 1, 0)

699 + else:

700 + start, end = frame_indices[0], frame_indices[-1]

701 + frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()

702 + elif len(frame_indices) > extract_t:

703 + frame_indices = np.linspace(

704 + 0, total_frames_num - 1, extract_t, dtype=int

705 + ).tolist()

706 +

707 + seen, uniq = set(), []

708 + for idx in frame_indices:

709 + if idx not in seen:

710 + seen.add(idx)

711 + uniq.append(idx)

712 +

713 + return uniq

714 +

715 + @classmethod

716 + def load_bytes(

717 + cls,

718 + data: bytes,

719 + num_frames: int = -1,

720 + fps: int = 2,

721 + max_duration: int = 300,

722 + frame_recovery: bool = False,

723 + *,

724 + backend: Literal["opencv", "pyav"] = "opencv",

725 + **kwargs,

726 + ) -> tuple[npt.NDArray, dict[str, Any]]:

727 + frames, metadata = super().load_bytes(

728 + data,

729 + num_frames=num_frames,

730 + fps=fps,

731 + max_duration=max_duration,

732 + frame_recovery=frame_recovery,

733 + backend=backend,

734 + **kwargs,

735 + )

736 + # Ensure even frame count — matches HF's sample_frames even-padding

737 + # and _preprocess temporal_patch_size divisibility check.

738 + if frames.shape[0] & 1:

739 + frames = np.concatenate([frames, frames[-1:]], axis=0)

740 + return frames, metadata

741 +

742 +

743 @VIDEO_LOADER_REGISTRY.register("molmo2")

744 class Molmo2VideoBackend(VideoLoader, OpenCVVideoBackendMixin):

745 @classmethod