[GLM4V] Avoid GLM4V processor init during startup metadata reads (#47155)
Signed-off-by: xiaoming <1259730330@qq.com>
labAxiaoming committed
Jul 4, 2026 at 06:03 UTC
d6d39c111e60e7b3c684503fe7a9bb7ecf9358dc
1 file changed
+67
-8
vllm/model_executor/models/glm4_1v.py
+67
-8
@@ -981,23 +981,50 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
981
def get_video_processor(self, **kwargs: object) -> Glm4vVideoProcessor:
982
return self.get_hf_processor(**kwargs).video_processor
983
984
+ def _get_processor_class_name(self) -> str | None:
985
+ from vllm.transformers_utils.processor import (
986
+ get_processor_cls_name_from_config,
987
+ )
988
+ from vllm.transformers_utils.utils import convert_model_repo_to_path
989
+
990
+ return get_processor_cls_name_from_config(
991
+ convert_model_repo_to_path(self.ctx.model_config.model),
992
+ revision=self.ctx.model_config.revision,
993
+ )
994
+
995
+ @staticmethod
996
+ def _get_longest_edge(size: Any, config_name: str) -> int:
997
+ if isinstance(size, dict):
998
+ longest_edge = size.get("longest_edge")
999
+ else:
1000
+ longest_edge = getattr(size, "longest_edge", None)
1001
+
1002
+ if longest_edge is None:
1003
+ raise ValueError(f"{config_name} must define longest_edge")
1004
+
1005
+ return int(longest_edge)
1006
+
1007
def get_mm_max_tokens_per_item(
1008
self,
1009
seq_len: int,
1010
mm_counts: Mapping[str, int],
1011
) -> Mapping[str, int] | None:
989
- processor = self.get_hf_processor()
990
- if isinstance(processor, Glm4vProcessor):
1012
+ processor_class_name = self._get_processor_class_name()
1013
+ if processor_class_name == "Glm4vProcessor":
1014
return None
1015
1016
+ if processor_class_name is None:
1017
+ processor = self.get_hf_processor()
1018
+ if isinstance(processor, Glm4vProcessor):
1019
+ return None
1020
+
1021
result: dict[str, int] = {}
1022
1023
if mm_counts.get("image", 0) > 0:
1024
result["image"] = self.get_max_image_tokens()
1025
1026
if mm_counts.get("video", 0) > 0:
999
- video_processor = self.get_video_processor()
1000
- max_pixels = video_processor.size["longest_edge"]
1027
+ max_pixels = self._get_video_max_pixels()
1028
1029
vision_config = self.get_hf_config().vision_config
1030
temporal_patch_size = vision_config.temporal_patch_size
@@ -1076,7 +1103,40 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
1103
``smart_resize`` as the ``max_pixels`` argument, which constrains
1104
``t_bar * h_bar * w_bar <= max_pixels``.
1105
"""
1079
- return self.get_image_processor().size["longest_edge"]
1106
+ mm_kwargs = self.ctx.get_merged_mm_kwargs({})
1107
+ if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
1108
+ return int(override_max_pixels)
1109
+
1110
+ image_processor_config = self.ctx.get_hf_image_processor_config()
1111
+ if not image_processor_config.get("size"):
1112
+ from transformers.image_processing_base import ImageProcessingMixin
1113
+
1114
+ image_processor_config, _ = ImageProcessingMixin.get_image_processor_dict(
1115
+ self.ctx.model_config.model,
1116
+ revision=self.ctx.model_config.revision,
1117
+ )
1118
+ size = image_processor_config["size"]
1119
+ if override_size := mm_kwargs.get("size"):
1120
+ size = size | override_size
1121
+
1122
+ return self._get_longest_edge(size, "GLM4V image processor size")
1123
+
1124
+ def _get_video_max_pixels(self) -> int:
1125
+ from transformers.video_processing_utils import BaseVideoProcessor
1126
+
1127
+ mm_kwargs = self.ctx.get_merged_mm_kwargs({})
1128
+ if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
1129
+ return int(override_max_pixels)
1130
+
1131
+ video_processor_config, _ = BaseVideoProcessor.get_video_processor_dict(
1132
+ self.ctx.model_config.model,
1133
+ revision=self.ctx.model_config.revision,
1134
+ )
1135
+ size = video_processor_config["size"]
1136
+ if override_size := mm_kwargs.get("size"):
1137
+ size = size | override_size
1138
+
1139
+ return self._get_longest_edge(size, "GLM4V video processor size")
1140
1141
def get_image_size_with_most_features(self) -> ImageSize:
1142
# Use num_frames=1 for single-image budget estimation.
@@ -1424,13 +1484,12 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
1484
num_videos = mm_counts.get("video", 0)
1485
1486
hf_config = self.info.get_hf_config()
1427
- hf_processor = self.info.get_hf_processor()
1487
tokenizer = self.info.get_tokenizer()
1488
1430
- image_token: str = hf_processor.image_token
1489
+ image_token = tokenizer.decode([hf_config.image_token_id])
1490
video_token_ids = [
1491
hf_config.video_start_token_id,
1433
- hf_processor.video_token_id,
1492
+ hf_config.video_token_id,
1493
hf_config.video_end_token_id,
1494
]
1495
video_token = tokenizer.decode(video_token_ids)