A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 16:30 UTC
Add GlmOcrConfig for GLM-OCR model type recognition (#34982)
hujia177 committed
Feb 26, 2026 at 09:04 UTC
ec8f943db1b8e5f3b32ed2ec29526b8a9a521088
3 files changed
+96
vllm/transformers_utils/config.py
+1
@@ -82,6 +82,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
82
deepseek_v32="DeepseekV3Config",
83
flex_olmo="FlexOlmoConfig",
84
funaudiochat="FunAudioChatConfig",
85
+ glm_ocr="GlmOcrConfig",
86
hunyuan_vl="HunYuanVLConfig",
87
isaac="IsaacConfig",
88
kimi_linear="KimiLinearConfig",
vllm/transformers_utils/configs/__init__.py
+4
@@ -28,6 +28,8 @@ _CLASS_TO_MODULE: dict[str, str] = {
28
"FlexOlmoConfig": "vllm.transformers_utils.configs.flex_olmo",
29
"FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat",
30
"FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat",
31
+ "GlmOcrConfig": "vllm.transformers_utils.configs.glm_ocr",
32
+ "GlmOcrVisionConfig": "vllm.transformers_utils.configs.glm_ocr",
33
"HunYuanVLConfig": "vllm.transformers_utils.configs.hunyuan_vl",
34
"HunYuanVLTextConfig": "vllm.transformers_utils.configs.hunyuan_vl",
35
"HunYuanVLVisionConfig": "vllm.transformers_utils.configs.hunyuan_vl",
@@ -83,6 +85,8 @@ __all__ = [
85
"FlexOlmoConfig",
86
"FunAudioChatConfig",
87
"FunAudioChatAudioEncoderConfig",
88
+ "GlmOcrConfig",
89
+ "GlmOcrVisionConfig",
90
"HunYuanVLConfig",
91
"HunYuanVLTextConfig",
92
"HunYuanVLVisionConfig",
vllm/transformers_utils/configs/glm_ocr.py
new
+91
@@ -0,0 +1,91 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from __future__ import annotations
4
+
5
+from typing import Any
6
+
7
+from transformers.configuration_utils import PretrainedConfig
8
+
9
+
10
+class GlmOcrVisionConfig(PretrainedConfig):
11
+ model_type = "glm_ocr_vision"
12
+
13
+ def __init__(
14
+ self,
15
+ hidden_size: int = 1024,
16
+ depth: int = 24,
17
+ num_heads: int = 16,
18
+ attention_bias: bool = True,
19
+ intermediate_size: int = 4096,
20
+ hidden_act: str = "silu",
21
+ hidden_dropout_prob: float = 0.0,
22
+ initializer_range: float = 0.02,
23
+ image_size: int = 336,
24
+ in_channels: int = 3,
25
+ patch_size: int = 14,
26
+ out_hidden_size: int = 1536,
27
+ rms_norm_eps: float = 1e-5,
28
+ spatial_merge_size: int = 2,
29
+ temporal_patch_size: int = 2,
30
+ **kwargs: Any,
31
+ ):
32
+ super().__init__(**kwargs)
33
+ self.hidden_size = hidden_size
34
+ self.depth = depth
35
+ self.num_heads = num_heads
36
+ self.attention_bias = attention_bias
37
+ self.intermediate_size = intermediate_size
38
+ self.hidden_act = hidden_act
39
+ self.hidden_dropout_prob = hidden_dropout_prob
40
+ self.initializer_range = initializer_range
41
+ self.image_size = image_size
42
+ self.in_channels = in_channels
43
+ self.patch_size = patch_size
44
+ self.out_hidden_size = out_hidden_size
45
+ self.rms_norm_eps = rms_norm_eps
46
+ self.spatial_merge_size = spatial_merge_size
47
+ self.temporal_patch_size = temporal_patch_size
48
+
49
+
50
+class GlmOcrConfig(PretrainedConfig):
51
+ model_type = "glm_ocr"
52
+
53
+ def __init__(
54
+ self,
55
+ text_config: dict | None = None,
56
+ vision_config: dict | None = None,
57
+ image_start_token_id: int = 59256,
58
+ image_end_token_id: int = 59257,
59
+ video_start_token_id: int = 59258,
60
+ video_end_token_id: int = 59259,
61
+ image_token_id: int = 59280,
62
+ video_token_id: int = 59281,
63
+ **kwargs: Any,
64
+ ):
65
+ super().__init__(**kwargs)
66
+ self.image_start_token_id = image_start_token_id
67
+ self.image_end_token_id = image_end_token_id
68
+ self.video_start_token_id = video_start_token_id
69
+ self.video_end_token_id = video_end_token_id
70
+ self.image_token_id = image_token_id
71
+ self.video_token_id = video_token_id
72
+ self.vision_config = GlmOcrVisionConfig(**(vision_config or {}))
73
+
74
+ if isinstance(text_config, dict):
75
+ from transformers import AutoConfig
76
+
77
+ model_type = text_config.get("model_type", "chatglm")
78
+ self.text_config = AutoConfig.for_model(model_type, **text_config)
79
+ elif text_config is None:
80
+ from transformers import AutoConfig
81
+
82
+ self.text_config = AutoConfig.for_model("chatglm")
83
+ else:
84
+ self.text_config = text_config
85
+
86
+ def get_text_config(self) -> PretrainedConfig:
87
+ return self.text_config
88
+
89
+ def save_pretrained(self, save_directory, **kwargs):
90
+ self._auto_class = None
91
+ super().save_pretrained(save_directory, **kwargs)