A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 14:45 UTC
[GLM-OCR] GLM-OCR with MTP Support (#33005)
Signed-off-by: zRzRzRzRzRzRzR <2448370773@qq.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Yuxuan Zhang committed
Jan 26, 2026 at 22:24 UTC
bb17e8f11c3835fbb930c1669c069d950bec110a
14 files changed
+873
-8
docs/models/supported_models.md
+1
@@ -674,6 +674,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
674
| `GLM4VForCausalLM`<sup>^</sup> | GLM-4V | T + I | `zai-org/glm-4v-9b`, `zai-org/cogagent-9b-20241220`, etc. | ✅︎ | ✅︎ |
675
| `Glm4vForConditionalGeneration` | GLM-4.1V-Thinking | T + I<sup>E+</sup> + V<sup>E+</sup> | `zai-org/GLM-4.1V-9B-Thinking`, etc. | ✅︎ | ✅︎ |
676
| `Glm4vMoeForConditionalGeneration` | GLM-4.5V | T + I<sup>E+</sup> + V<sup>E+</sup> | `zai-org/GLM-4.5V`, etc. | ✅︎ | ✅︎ |
677
+| `GlmOcrForConditionalGeneration` | GLM-OCR | T + I<sup>E+</sup> | `zai-org/GLM-OCR`, etc. | ✅︎ | ✅︎ |
678
| `GraniteSpeechForConditionalGeneration` | Granite Speech | T + A | `ibm-granite/granite-speech-3.3-8b` | ✅︎ | ✅︎ |
679
| `H2OVLChatModel` | H2OVL | T + I<sup>E+</sup> | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | | ✅︎ |
680
| `HunYuanVLForConditionalGeneration` | HunyuanOCR | T + I<sup>E+</sup> | `tencent/HunyuanOCR`, etc. | ✅︎ | ✅︎ |
examples/offline_inference/vision_language.py
+38
@@ -566,6 +566,42 @@ def run_glm4_5v_fp8(questions: list[str], modality: str) -> ModelRequestData:
566
)
567
568
569
+# GLM-OCR
570
+def run_glm_ocr(questions: list[str], modality: str) -> ModelRequestData:
571
+ model_name = "zai-org/GLM-OCR"
572
+
573
+ engine_args = EngineArgs(
574
+ model=model_name,
575
+ max_model_len=4096,
576
+ max_num_seqs=2,
577
+ mm_processor_kwargs={
578
+ "size": {"shortest_edge": 12544, "longest_edge": 47040000},
579
+ "fps": 1,
580
+ },
581
+ limit_mm_per_prompt={modality: 1},
582
+ enforce_eager=True,
583
+ )
584
+
585
+ if modality == "image":
586
+ placeholder = "<|begin_of_image|><|image|><|end_of_image|>"
587
+ elif modality == "video":
588
+ placeholder = "<|begin_of_video|><|video|><|end_of_video|>"
589
+
590
+ prompts = [
591
+ (
592
+ "[gMASK]<sop><|system|>\nYou are a helpful assistant.<|user|>\n"
593
+ f"{placeholder}"
594
+ f"{question}<|assistant|>assistant\n"
595
+ )
596
+ for question in questions
597
+ ]
598
+
599
+ return ModelRequestData(
600
+ engine_args=engine_args,
601
+ prompts=prompts,
602
+ )
603
+
604
+
605
# H2OVL-Mississippi
606
def run_h2ovl(questions: list[str], modality: str) -> ModelRequestData:
607
assert modality == "image"
@@ -1988,6 +2024,7 @@ model_example_map = {
2024
"glm4_1v": run_glm4_1v,
2025
"glm4_5v": run_glm4_5v,
2026
"glm4_5v_fp8": run_glm4_5v_fp8,
2027
+ "glm_ocr": run_glm_ocr,
2028
"h2ovl_chat": run_h2ovl,
2029
"hunyuan_vl": run_hunyuan_vl,
2030
"hyperclovax_seed_vision": run_hyperclovax_seed_vision,
@@ -2040,6 +2077,7 @@ model_example_map = {
2077
2078
MODELS_NEED_VIDEO_METADATA = [
2079
"glm4_1v",
2080
+ "glm_ocr",
2081
"glm4_5v",
2082
"glm4_5v_fp8",
2083
"molmo2",
tests/models/multimodal/generation/test_common.py
+14
@@ -458,6 +458,20 @@ VLM_TEST_SETTINGS = {
458
],
459
marks=[large_gpu_mark(min_gb=32)],
460
),
461
+ "glm_ocr": VLMTestInfo(
462
+ models=["zai-org/GLM-OCR"],
463
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
464
+ prompt_formatter=lambda img_prompt: f"[gMASK]<|user|>\n{img_prompt}<|assistant|>\n", # noqa: E501
465
+ img_idx_to_prompt=lambda idx: "<|begin_of_image|><|image|><|end_of_image|>",
466
+ video_idx_to_prompt=lambda idx: "<|begin_of_video|><|video|><|end_of_video|>",
467
+ max_model_len=2048,
468
+ max_num_seqs=2,
469
+ get_stop_token_ids=lambda tok: [151329, 151336, 151338],
470
+ num_logprobs=10,
471
+ image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
472
+ auto_cls=AutoModelForImageTextToText,
473
+ marks=[large_gpu_mark(min_gb=32)],
474
+ ),
475
"h2ovl": VLMTestInfo(
476
models=[
477
"h2oai/h2ovl-mississippi-800m",
tests/models/multimodal/generation/test_vit_backend_functionality.py
+13
@@ -91,6 +91,19 @@ MODEL_CONFIGS: dict[str, dict[str, Any]] = {
91
"use_processor": True,
92
"question": "What is the content of each image?",
93
},
94
+ "glm_ocr": {
95
+ "model_name": "zai-org/GLM-OCR",
96
+ "interface": "llm_generate",
97
+ "max_model_len": 131072,
98
+ "max_num_seqs": 2,
99
+ "sampling_params": {
100
+ "temperature": 0.0,
101
+ "max_tokens": 256,
102
+ "stop_token_ids": None,
103
+ },
104
+ "use_processor": True,
105
+ "question": "Text Recognition:",
106
+ },
107
"keye_vl": {
108
"model_name": "Kwai-Keye/Keye-VL-8B-Preview",
109
"interface": "llm_generate",
tests/models/multimodal/processing/test_common.py
+1
@@ -122,6 +122,7 @@ MM_DATA_PATCHES = {
122
"ernie4_5_moe_vl": qwen3_vl_patch_mm_data,
123
"glm4v": glm4_1v_patch_mm_data,
124
"glm4v_moe": glm4_1v_patch_mm_data,
125
+ "glm_ocr": glm4_1v_patch_mm_data,
126
"glmasr": glmasr_patch_mm_data,
127
"molmo2": qwen3_vl_patch_mm_data,
128
"qwen3_vl": qwen3_vl_patch_mm_data,
tests/models/registry.py
+11
-1
@@ -274,7 +274,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
274
"Glm4MoeLiteForCausalLM": _HfExamplesInfo(
275
"zai-org/GLM-4.7-Flash",
276
min_transformers_version="5.0.0.dev",
277
- is_available_online=False,
277
),
278
"GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2", {"alias": "gpt2"}),
279
"GPTBigCodeForCausalLM": _HfExamplesInfo(
@@ -707,6 +706,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
706
),
707
"Glm4vForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.1V-9B-Thinking"),
708
"Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V"),
709
+ "GlmOcrForConditionalGeneration": _HfExamplesInfo(
710
+ "zai-org/GLM-OCR",
711
+ is_available_online=False,
712
+ min_transformers_version="5.0.0.dev",
713
+ ),
714
"H2OVLChatModel": _HfExamplesInfo(
715
"h2oai/h2ovl-mississippi-800m",
716
trust_remote_code=True,
@@ -1053,7 +1057,13 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
1057
"Glm4MoeLiteMTPModel": _HfExamplesInfo(
1058
"zai-org/GLM-4.7-Flash",
1059
speculative_model="zai-org/GLM-4.7-Flash",
1060
+ min_transformers_version="5.0.0.dev",
1061
+ ),
1062
+ "GlmOcrMTPModel": _HfExamplesInfo(
1063
+ "zai-org/GLM-OCR",
1064
+ speculative_model="zai-org/GLM-OCR",
1065
is_available_online=False,
1066
+ min_transformers_version="5.0.0.dev",
1067
),
1068
"LongCatFlashMTPModel": _HfExamplesInfo(
1069
"meituan-longcat/LongCat-Flash-Chat",
vllm/config/speculative.py
+12
@@ -34,6 +34,7 @@ MTPModelTypes = Literal[
34
"mimo_mtp",
35
"glm4_moe_mtp",
36
"glm4_moe_lite_mtp",
37
+ "glm_ocr_mtp",
38
"ernie_mtp",
39
"exaone_moe_mtp",
40
"qwen3_next_mtp",
@@ -221,6 +222,17 @@ class SpeculativeConfig:
222
}
223
)
224
225
+ if hf_config.architectures[0] == "GlmOcrForConditionalGeneration":
226
+ hf_config.model_type = "glm_ocr_mtp"
227
+ n_predict = getattr(hf_config, "num_nextn_predict_layers", None)
228
+ hf_config.update(
229
+ {
230
+ "num_hidden_layers": 0,
231
+ "n_predict": n_predict,
232
+ "architectures": ["GlmOcrMTPModel"],
233
+ }
234
+ )
235
+
236
if hf_config.model_type == "ernie4_5_moe":
237
hf_config.model_type = "ernie_mtp"
238
if hf_config.model_type == "ernie_mtp":
vllm/model_executor/models/glm4.py
+99
-2
@@ -39,13 +39,22 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
39
from vllm.model_executor.layers.quantization import QuantizationConfig
40
from vllm.model_executor.layers.rotary_embedding import get_rope
41
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
42
+from vllm.model_executor.model_loader.weight_utils import (
43
+ default_weight_loader,
44
+ maybe_remap_kv_scale_name,
45
+)
46
from vllm.sequence import IntermediateTensors
47
from vllm.v1.attention.backend import AttentionType
48
49
from .interfaces import SupportsLoRA, SupportsPP
50
from .llama import LlamaMLP as Glm4MLP
51
from .llama import LlamaModel
48
-from .utils import AutoWeightsLoader, PPMissingLayer, maybe_prefix
52
+from .utils import (
53
+ AutoWeightsLoader,
54
+ PPMissingLayer,
55
+ is_pp_missing_parameter,
56
+ maybe_prefix,
57
+)
58
59
60
class Glm4Attention(nn.Module):
@@ -78,7 +87,15 @@ class Glm4Attention(nn.Module):
87
# Number of KV heads is less than TP size, so we replicate
88
# the KV heads across multiple tensor parallel GPUs.
89
assert tp_size % self.total_num_kv_heads == 0
81
- config.rope_parameters.setdefault("partial_rotary_factor", 0.5)
90
+
91
+ rope_params = getattr(config, "rope_parameters", None)
92
+ if isinstance(rope_params, dict) and "partial_rotary_factor" in rope_params:
93
+ config.rope_parameters.setdefault(
94
+ "partial_rotary_factor", rope_params["partial_rotary_factor"]
95
+ )
96
+ else:
97
+ config.rope_parameters.setdefault("partial_rotary_factor", 0.5)
98
+
99
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
100
self.head_dim = head_dim or hidden_size // self.total_num_heads
101
self.q_size = self.num_heads * self.head_dim
@@ -220,6 +237,73 @@ class Glm4Model(LlamaModel):
237
vllm_config=vllm_config, prefix=prefix, layer_type=Glm4DecoderLayer
238
)
239
240
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
241
+ stacked_params_mapping = [
242
+ # (param_name, shard_name, shard_id)
243
+ (".qkv_proj", ".q_proj", "q"),
244
+ (".qkv_proj", ".k_proj", "k"),
245
+ (".qkv_proj", ".v_proj", "v"),
246
+ (".gate_up_proj", ".gate_proj", 0),
247
+ (".gate_up_proj", ".up_proj", 1),
248
+ ]
249
+ params_dict = dict(self.named_parameters())
250
+ loaded_params: set[str] = set()
251
+ for name, loaded_weight in weights:
252
+ spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
253
+ if spec_layer is not None:
254
+ continue
255
+ if "rotary_emb.inv_freq" in name:
256
+ continue
257
+ if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
258
+ # Models trained using ColossalAI may include these tensors in
259
+ # the checkpoint. Skip them.
260
+ continue
261
+ if self.quant_config is not None and (
262
+ scale_name := self.quant_config.get_cache_scale(name)
263
+ ):
264
+ # Loading kv cache quantization scales
265
+ param = params_dict[scale_name]
266
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
267
+ loaded_weight = (
268
+ loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]
269
+ )
270
+ weight_loader(param, loaded_weight)
271
+ loaded_params.add(scale_name)
272
+ continue
273
+ if "scale" in name or "zero_point" in name:
274
+ # Remapping the name of FP8 kv-scale or zero point.
275
+ name = maybe_remap_kv_scale_name(name, params_dict)
276
+ if name is None:
277
+ continue
278
+ for param_name, weight_name, shard_id in stacked_params_mapping:
279
+ if weight_name not in name:
280
+ continue
281
+ name = name.replace(weight_name, param_name)
282
+ # Skip loading extra bias for GPTQ models.
283
+ if name.endswith(".bias") and name not in params_dict:
284
+ continue
285
+
286
+ if is_pp_missing_parameter(name, self):
287
+ continue
288
+
289
+ param = params_dict[name]
290
+ weight_loader = param.weight_loader
291
+ weight_loader(param, loaded_weight, shard_id)
292
+ break
293
+ else:
294
+ # Skip loading extra bias for GPTQ models.
295
+ if name.endswith(".bias") and name not in params_dict:
296
+ continue
297
+
298
+ if is_pp_missing_parameter(name, self):
299
+ continue
300
+
301
+ param = params_dict[name]
302
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
303
+ weight_loader(param, loaded_weight)
304
+ loaded_params.add(name)
305
+ return loaded_params
306
+
307
308
class Glm4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
309
packed_modules_mapping = {
@@ -293,3 +377,16 @@ class Glm4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
377
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
378
)
379
return loader.load_weights(weights)
380
+
381
+
382
+def get_spec_layer_idx_from_weight_name(
383
+ config: Glm4Config, weight_name: str
384
+) -> int | None:
385
+ if hasattr(config, "num_nextn_predict_layers") and (
386
+ config.num_nextn_predict_layers > 0
387
+ ):
388
+ layer_idx = config.num_hidden_layers
389
+ for i in range(config.num_nextn_predict_layers):
390
+ if f"layers.{layer_idx + i}." in weight_name:
391
+ return layer_idx + i
392
+ return None
vllm/model_executor/models/glm4_1v.py
+3
-2
@@ -24,7 +24,8 @@
24
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
25
# See the License for the specific language governing permissions and
26
# limitations under the License.
27
-"""Inference-only GLM-4V model compatible with HuggingFace weights."""
27
+"""Inference-only GLM-4.1V & GLM-4.6V-Flash, AutoGLM-Phone-9B model
28
+compatible with HuggingFace weights."""
29
30
import itertools
31
import math
@@ -1418,7 +1419,7 @@ class Glm4vForConditionalGeneration(
1419
prefix=maybe_prefix(prefix, "visual"),
1420
)
1421
1421
- if config.model_type == "glm4v":
1422
+ if config.model_type in ("glm4v", "glm_ocr"):
1423
architectures = ["Glm4ForCausalLM"]
1424
elif config.model_type == "glm4v_moe":
1425
architectures = ["Glm4MoeForCausalLM"]
vllm/model_executor/models/glm_ocr.py
new
+389
@@ -0,0 +1,389 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Adapted from
5
+# https://github.com/huggingface/transformers/blob/main/src/transformers/models/Glm4v/modeling_Glm4v.py
6
+# Copyright 2026 The ZhipuAI Team.
7
+# Copyright 2026 The vLLM team.
8
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
9
+# All rights reserved.
10
+#
11
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
12
+# and OPT implementations in this library. It has been modified from its
13
+# original forms to accommodate minor architectural differences compared
14
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
15
+#
16
+# Licensed under the Apache License, Version 2.0 (the "License");
17
+# you may not use this file except in compliance with the License.
18
+# You may obtain a copy of the License at
19
+#
20
+# http://www.apache.org/licenses/LICENSE-2.0
21
+#
22
+# Unless required by applicable law or agreed to in writing, software
23
+# distributed under the License is distributed on an "AS IS" BASIS,
24
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
25
+# See the License for the specific language governing permissions and
26
+# limitations under the License.
27
+"""Inference-only GLM-OCR model compatible with HuggingFace weights."""
28
+
29
+from collections.abc import Callable
30
+from functools import partial
31
+from typing import TYPE_CHECKING
32
+
33
+import torch
34
+import torch.nn as nn
35
+from einops import rearrange
36
+
37
+if TYPE_CHECKING:
38
+ from transformers.models.glm_ocr.configuration_glm_ocr import GlmOcrVisionConfig
39
+
40
+from vllm.config import VllmConfig
41
+from vllm.distributed import get_tensor_model_parallel_world_size, parallel_state
42
+from vllm.distributed import utils as dist_utils
43
+from vllm.logger import init_logger
44
+from vllm.model_executor.layers.attention.mm_encoder_attention import (
45
+ MMEncoderAttention,
46
+)
47
+from vllm.model_executor.layers.conv import Conv2dLayer
48
+from vllm.model_executor.layers.layernorm import RMSNorm
49
+from vllm.model_executor.layers.linear import (
50
+ QKVParallelLinear,
51
+ RowParallelLinear,
52
+)
53
+from vllm.model_executor.layers.quantization import QuantizationConfig
54
+from vllm.model_executor.layers.rotary_embedding import get_rope
55
+from vllm.model_executor.layers.rotary_embedding.common import (
56
+ ApplyRotaryEmb,
57
+)
58
+from vllm.model_executor.models.glm4_1v import (
59
+ Glm4vDummyInputsBuilder,
60
+ Glm4vForConditionalGeneration,
61
+ Glm4vMultiModalProcessor,
62
+ Glm4vPatchMerger,
63
+ Glm4vProcessingInfo,
64
+ Glm4vVisionBlock,
65
+ Glm4vVisionMLP,
66
+ Glm4vVisionPatchEmbed,
67
+ Glm4vVisionTransformer,
68
+)
69
+from vllm.multimodal import MULTIMODAL_REGISTRY
70
+
71
+from .utils import (
72
+ maybe_prefix,
73
+)
74
+from .vision import (
75
+ get_vit_attn_backend,
76
+ is_vit_use_data_parallel,
77
+)
78
+
79
+logger = init_logger(__name__)
80
+
81
+
82
+class GlmOcrVisionMLP(Glm4vVisionMLP):
83
+ pass
84
+
85
+
86
+class GlmOcrVisionAttention(nn.Module):
87
+ def __init__(
88
+ self,
89
+ embed_dim: int,
90
+ num_heads: int,
91
+ projection_size: int,
92
+ quant_config: QuantizationConfig | None = None,
93
+ prefix: str = "",
94
+ ) -> None:
95
+ super().__init__()
96
+ # Per attention head and per partition values.
97
+ use_data_parallel = is_vit_use_data_parallel()
98
+ self.tp_size = (
99
+ 1 if use_data_parallel else get_tensor_model_parallel_world_size()
100
+ )
101
+ self.tp_rank = (
102
+ 0 if use_data_parallel else parallel_state.get_tensor_model_parallel_rank()
103
+ )
104
+ self.hidden_size_per_attention_head = dist_utils.divide(
105
+ projection_size, num_heads
106
+ )
107
+ self.num_attention_heads_per_partition = dist_utils.divide(
108
+ num_heads, self.tp_size
109
+ )
110
+
111
+ self.head_dim = embed_dim // num_heads
112
+
113
+ self.q_norm = RMSNorm(self.head_dim, eps=1e-5)
114
+ self.k_norm = RMSNorm(self.head_dim, eps=1e-5)
115
+
116
+ self.qkv = QKVParallelLinear(
117
+ hidden_size=embed_dim,
118
+ head_size=self.hidden_size_per_attention_head,
119
+ total_num_heads=num_heads,
120
+ total_num_kv_heads=num_heads,
121
+ bias=True,
122
+ quant_config=quant_config,
123
+ # Change qkv prefix to align with GLM-4.5V-FP8 quantization cfg
124
+ prefix=f"{prefix}.qkv_proj" if quant_config else f"{prefix}.qkv",
125
+ disable_tp=use_data_parallel,
126
+ )
127
+ self.proj = RowParallelLinear(
128
+ input_size=projection_size,
129
+ output_size=embed_dim,
130
+ quant_config=quant_config,
131
+ prefix=f"{prefix}.proj",
132
+ bias=True,
133
+ disable_tp=use_data_parallel,
134
+ )
135
+
136
+ self.attn = MMEncoderAttention(
137
+ num_heads=self.num_attention_heads_per_partition,
138
+ head_size=self.hidden_size_per_attention_head,
139
+ scale=self.hidden_size_per_attention_head**-0.5,
140
+ )
141
+ self.apply_rotary_emb = ApplyRotaryEmb(enforce_enable=True)
142
+
143
+ def split_qkv(self, qkv: torch.Tensor) -> tuple[torch.Tensor, ...]:
144
+ # [s, b, 3 * head * head_dim]
145
+ seq_len, bs, _ = qkv.shape
146
+
147
+ # [s, b, 3 * head * head_dim] -> 3 * [s, b, head * head_dim]
148
+ q, k, v = qkv.chunk(3, dim=2)
149
+
150
+ # 3 * [s, b, head * head_dim] -> 3 * [s, b, head, head_dim]
151
+ new_shape = (
152
+ seq_len,
153
+ bs,
154
+ self.num_attention_heads_per_partition,
155
+ self.hidden_size_per_attention_head,
156
+ )
157
+ q, k, v = (x.view(*new_shape) for x in (q, k, v))
158
+ return q, k, v
159
+
160
+ def forward(
161
+ self,
162
+ x: torch.Tensor,
163
+ cu_seqlens: torch.Tensor,
164
+ rotary_pos_emb_cos: torch.Tensor,
165
+ rotary_pos_emb_sin: torch.Tensor,
166
+ max_seqlen: torch.Tensor | None = None, # Only used for Flash Attention
167
+ ) -> torch.Tensor:
168
+ # [s, b, c] --> [s, b, head * 3 * head_dim]
169
+ x, _ = self.qkv(x)
170
+
171
+ # [s, b, 3 * head * head_dim] -> 3 * [s, b, head, head_dim]
172
+ q, k, v = self.split_qkv(x)
173
+
174
+ # RMSNorm on q, k
175
+ q_shape, k_shape = q.shape, k.shape
176
+ q = self.q_norm(q.reshape(-1, self.head_dim)).view(q_shape)
177
+ k = self.k_norm(k.reshape(-1, self.head_dim)).view(k_shape)
178
+
179
+ q, k, v = (rearrange(x, "s b ... -> b s ...").contiguous() for x in (q, k, v))
180
+ if rotary_pos_emb_cos is not None and rotary_pos_emb_sin is not None:
181
+ # [2 * b, s, heads, head_dim]
182
+ qk_concat = torch.cat([q, k], dim=0)
183
+ qk_rotated = self.apply_rotary_emb(
184
+ qk_concat,
185
+ rotary_pos_emb_cos,
186
+ rotary_pos_emb_sin,
187
+ )
188
+ q, k = torch.chunk(qk_rotated, 2, dim=0)
189
+
190
+ context_layer = self.attn(
191
+ query=q,
192
+ key=k,
193
+ value=v,
194
+ cu_seqlens=cu_seqlens,
195
+ max_seqlen=max_seqlen,
196
+ )
197
+ context_layer = rearrange(context_layer, "b s h d -> s b (h d)").contiguous()
198
+
199
+ output, _ = self.proj(context_layer)
200
+ return output
201
+
202
+
203
+class GlmOcrVisionBlock(Glm4vVisionBlock):
204
+ def __init__(
205
+ self,
206
+ dim: int,
207
+ num_heads: int,
208
+ mlp_hidden_dim: int,
209
+ norm_layer: Callable[[int], nn.Module] | None = None,
210
+ quant_config: QuantizationConfig | None = None,
211
+ prefix: str = "",
212
+ ) -> None:
213
+ super().__init__(
214
+ dim,
215
+ num_heads,
216
+ mlp_hidden_dim,
217
+ norm_layer,
218
+ quant_config,
219
+ prefix,
220
+ )
221
+ if norm_layer is None:
222
+ norm_layer = partial(nn.LayerNorm, eps=1e-6)
223
+ self.norm1 = norm_layer(dim)
224
+ self.norm2 = norm_layer(dim)
225
+ self.attn = GlmOcrVisionAttention(
226
+ embed_dim=dim,
227
+ num_heads=num_heads,
228
+ projection_size=dim,
229
+ quant_config=quant_config,
230
+ prefix=f"{prefix}.attn",
231
+ )
232
+ self.mlp = GlmOcrVisionMLP(
233
+ dim,
234
+ mlp_hidden_dim,
235
+ bias=True,
236
+ quant_config=quant_config,
237
+ prefix=f"{prefix}.mlp",
238
+ )
239
+
240
+
241
+class GlmOcrVisionPatchEmbed(Glm4vVisionPatchEmbed):
242
+ pass
243
+
244
+
245
+class GlmOcrPatchMerger(Glm4vPatchMerger):
246
+ pass
247
+
248
+
249
+class GlmOcrVisionTransformer(Glm4vVisionTransformer):
250
+ def __init__(
251
+ self,
252
+ vision_config: GlmOcrVisionConfig,
253
+ norm_eps: float = 1e-5,
254
+ quant_config: QuantizationConfig | None = None,
255
+ prefix: str = "",
256
+ ) -> None:
257
+ super().__init__(vision_config, norm_eps, quant_config, prefix)
258
+
259
+ del self.post_conv_layernorm
260
+ del self.embeddings
261
+
262
+ patch_size = vision_config.patch_size
263
+ temporal_patch_size = vision_config.temporal_patch_size
264
+ in_channels = vision_config.in_channels
265
+ depth = vision_config.depth
266
+ self.hidden_size = vision_config.hidden_size
267
+ self.num_heads = vision_config.num_heads
268
+
269
+ self.patch_size = vision_config.patch_size
270
+ self.spatial_merge_size = vision_config.spatial_merge_size
271
+ self.out_hidden_size = vision_config.out_hidden_size
272
+
273
+ self.patch_embed = Glm4vVisionPatchEmbed(
274
+ patch_size=patch_size,
275
+ temporal_patch_size=temporal_patch_size,
276
+ in_channels=in_channels,
277
+ hidden_size=self.hidden_size,
278
+ )
279
+
280
+ norm_layer = partial(RMSNorm, eps=norm_eps)
281
+ head_dim = self.hidden_size // self.num_heads
282
+ self.rotary_pos_emb = get_rope(
283
+ head_size=head_dim,
284
+ max_position=8192,
285
+ is_neox_style=True,
286
+ rope_parameters={"partial_rotary_factor": 0.5},
287
+ )
288
+ self.blocks = nn.ModuleList(
289
+ [
290
+ GlmOcrVisionBlock(
291
+ dim=self.hidden_size,
292
+ num_heads=self.num_heads,
293
+ mlp_hidden_dim=vision_config.intermediate_size,
294
+ norm_layer=norm_layer,
295
+ quant_config=quant_config,
296
+ prefix=f"{prefix}.blocks.{layer_idx}",
297
+ )
298
+ for layer_idx in range(depth)
299
+ ]
300
+ )
301
+ self.merger = GlmOcrPatchMerger(
302
+ d_model=vision_config.out_hidden_size,
303
+ context_dim=vision_config.out_hidden_size * vision_config.in_channels,
304
+ quant_config=quant_config,
305
+ bias=False,
306
+ prefix=f"{prefix}.merger",
307
+ )
308
+
309
+ self.downsample = Conv2dLayer(
310
+ in_channels=vision_config.hidden_size,
311
+ out_channels=vision_config.out_hidden_size,
312
+ kernel_size=vision_config.spatial_merge_size,
313
+ stride=vision_config.spatial_merge_size,
314
+ )
315
+ self.post_layernorm = RMSNorm(
316
+ vision_config.hidden_size, eps=vision_config.rms_norm_eps
317
+ )
318
+
319
+ self.attn_backend = get_vit_attn_backend(
320
+ head_size=head_dim,
321
+ dtype=torch.get_default_dtype(),
322
+ )
323
+
324
+ def forward(
325
+ self,
326
+ x: torch.Tensor,
327
+ grid_thw: torch.Tensor | list[list[int]],
328
+ ) -> torch.Tensor:
329
+ if isinstance(grid_thw, list):
330
+ grid_thw = torch.tensor(grid_thw, dtype=torch.int32)
331
+
332
+ # patchify
333
+ x = x.to(device=self.device, dtype=self.dtype)
334
+ x = self.patch_embed(x)
335
+
336
+ # compute position embedding
337
+ rotary_pos_emb_cos, rotary_pos_emb_sin, image_type_ids = self.rot_pos_emb(
338
+ grid_thw
339
+ )
340
+ # compute cu_seqlens
341
+ cu_seqlens = torch.repeat_interleave(
342
+ grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]
343
+ ).cumsum(dim=0, dtype=torch.int32)
344
+ cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
345
+ cu_seqlens = cu_seqlens.to(self.device, non_blocking=True)
346
+
347
+ # pre-compute max_seqlen for attn mask to reduce cuMemcpy operations
348
+ max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
349
+
350
+ # transformers
351
+ x = x.unsqueeze(1)
352
+ for blk in self.blocks:
353
+ x = blk(
354
+ x,
355
+ cu_seqlens=cu_seqlens,
356
+ rotary_pos_emb_cos=rotary_pos_emb_cos,
357
+ rotary_pos_emb_sin=rotary_pos_emb_sin,
358
+ max_seqlen=max_seqlen,
359
+ )
360
+
361
+ # adapter
362
+ x = self.post_layernorm(x)
363
+
364
+ x = x.view(-1, self.spatial_merge_size, self.spatial_merge_size, x.shape[-1])
365
+ x = x.permute(0, 3, 1, 2)
366
+ x = self.downsample(x).view(-1, self.out_hidden_size)
367
+ x = self.merger(x)
368
+
369
+ return x
370
+
371
+
372
+@MULTIMODAL_REGISTRY.register_processor(
373
+ Glm4vMultiModalProcessor,
374
+ info=Glm4vProcessingInfo,
375
+ dummy_inputs=Glm4vDummyInputsBuilder,
376
+)
377
+class GlmOcrForConditionalGeneration(Glm4vForConditionalGeneration):
378
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
379
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
380
+ config = vllm_config.model_config.hf_config
381
+ quant_config = vllm_config.quant_config
382
+
383
+ with self._mark_tower_model(vllm_config, {"image", "video"}):
384
+ self.visual = GlmOcrVisionTransformer(
385
+ config.vision_config,
386
+ norm_eps=getattr(config, "rms_norm_eps", 1e-5),
387
+ quant_config=quant_config,
388
+ prefix=maybe_prefix(prefix, "visual"),
389
+ )
vllm/model_executor/models/glm_ocr_mtp.py
new
+285
@@ -0,0 +1,285 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Copyright 2026 The ZhipuAI Team.
5
+# Copyright 2026 The vLLM team.
6
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
7
+#
8
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
9
+# and OPT implementations in this library. It has been modified from its
10
+# original forms to accommodate minor architectural differences compared
11
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
12
+#
13
+# Licensed under the Apache License, Version 2.0 (the "License");
14
+# you may not use this file except in compliance with the License.
15
+# You may obtain a copy of the License at
16
+#
17
+# http://www.apache.org/licenses/LICENSE-2.0
18
+#
19
+# Unless required by applicable law or agreed to in writing, software
20
+# distributed under the License is distributed on an "AS IS" BASIS,
21
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22
+# See the License for the specific language governing permissions and
23
+# limitations under the License.
24
+"""Inference-only GLM-OCR MTP model compatible with HuggingFace weights."""
25
+
26
+from collections.abc import Iterable
27
+
28
+import torch
29
+import torch.nn as nn
30
+
31
+from vllm.config import VllmConfig
32
+from vllm.model_executor.layers.layernorm import RMSNorm
33
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
+from vllm.model_executor.layers.vocab_parallel_embedding import (
35
+ VocabParallelEmbedding,
36
+)
37
+from vllm.model_executor.model_loader.weight_utils import (
38
+ default_weight_loader,
39
+ maybe_remap_kv_scale_name,
40
+)
41
+from vllm.platforms import current_platform
42
+from vllm.sequence import IntermediateTensors
43
+
44
+from .glm4 import Glm4DecoderLayer, get_spec_layer_idx_from_weight_name
45
+from .glm4_moe_lite_mtp import (
46
+ Glm4MoeLiteMultiTokenPredictor,
47
+ SharedHead,
48
+)
49
+from .interfaces import SupportsPP
50
+from .utils import (
51
+ is_pp_missing_parameter,
52
+ maybe_prefix,
53
+)
54
+
55
+
56
+class GlmOcrMultiTokenPredictorLayer(nn.Module):
57
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
58
+ nn.Module.__init__(self)
59
+
60
+ config = vllm_config.speculative_config.draft_model_config.hf_config.text_config
61
+ self.config = config
62
+ quant_config = vllm_config.quant_config
63
+
64
+ self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
65
+ self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
66
+ self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
67
+
68
+ self.device = current_platform.device_type
69
+ self.shared_head = SharedHead(
70
+ config=config, prefix=prefix, quant_config=quant_config
71
+ )
72
+ self.mtp_block = Glm4DecoderLayer(
73
+ vllm_config=vllm_config, prefix=prefix, config=self.config
74
+ )
75
+
76
+ def forward(
77
+ self,
78
+ input_ids: torch.Tensor,
79
+ positions: torch.Tensor,
80
+ previous_hidden_states: torch.Tensor,
81
+ inputs_embeds: torch.Tensor | None = None,
82
+ spec_step_index: int = 0,
83
+ ) -> torch.Tensor:
84
+ assert inputs_embeds is not None
85
+ # masking inputs at position 0, as not needed by MTP
86
+ inputs_embeds[positions[0] == 0] = 0
87
+
88
+ inputs_embeds = self.enorm(inputs_embeds)
89
+ previous_hidden_states = self.hnorm(previous_hidden_states)
90
+
91
+ hidden_states = self.eh_proj(
92
+ torch.cat([inputs_embeds, previous_hidden_states], dim=-1)
93
+ )
94
+
95
+ hidden_states, residual = self.mtp_block(
96
+ positions=positions, hidden_states=hidden_states, residual=None
97
+ )
98
+ hidden_states = residual + hidden_states
99
+ return hidden_states
100
+
101
+
102
+class GlmOcrMultiTokenPredictor(Glm4MoeLiteMultiTokenPredictor):
103
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
104
+ nn.Module.__init__(self)
105
+ config = vllm_config.model_config.hf_config.text_config
106
+ self.mtp_start_layer_idx = config.num_hidden_layers
107
+ self.num_mtp_layers = config.num_nextn_predict_layers
108
+ self.layers = torch.nn.ModuleDict(
109
+ {
110
+ str(idx): GlmOcrMultiTokenPredictorLayer(
111
+ vllm_config=vllm_config,
112
+ prefix=f"{prefix}.layers.{idx}",
113
+ )
114
+ for idx in range(
115
+ self.mtp_start_layer_idx,
116
+ self.mtp_start_layer_idx + self.num_mtp_layers,
117
+ )
118
+ }
119
+ )
120
+ self.embed_tokens = VocabParallelEmbedding(
121
+ config.vocab_size,
122
+ config.hidden_size,
123
+ )
124
+ self.logits_processor = LogitsProcessor(config.vocab_size)
125
+
126
+
127
+class GlmOcrMTP(nn.Module, SupportsPP):
128
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
129
+ super().__init__()
130
+ self.config = vllm_config.model_config.hf_config.text_config
131
+ quant_config = vllm_config.quant_config
132
+ self.quant_config = quant_config
133
+ self.model = GlmOcrMultiTokenPredictor(
134
+ vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
135
+ )
136
+
137
+ self.expert_weights = []
138
+ self.num_layers = self.config.num_nextn_predict_layers
139
+ for layer in self.model.layers.values():
140
+ assert isinstance(layer, GlmOcrMultiTokenPredictorLayer)
141
+ layer = layer.mtp_block
142
+ assert isinstance(layer, Glm4DecoderLayer)
143
+
144
+ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
145
+ return self.model.embed_input_ids(input_ids)
146
+
147
+ def forward(
148
+ self,
149
+ input_ids: torch.Tensor,
150
+ positions: torch.Tensor,
151
+ hidden_states: torch.Tensor,
152
+ intermediate_tensors: IntermediateTensors | None = None,
153
+ inputs_embeds: torch.Tensor | None = None,
154
+ spec_step_idx: int = 0,
155
+ ) -> torch.Tensor:
156
+ hidden_states = self.model(
157
+ input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
158
+ )
159
+ return hidden_states
160
+
161
+ def compute_logits(
162
+ self,
163
+ hidden_states: torch.Tensor,
164
+ spec_step_idx: int = 0,
165
+ ) -> torch.Tensor | None:
166
+ return self.model.compute_logits(hidden_states, spec_step_idx)
167
+
168
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
169
+ stacked_params_mapping = [
170
+ # (param_name, shard_name, shard_id)
171
+ (".qkv_proj", ".q_proj", "q"),
172
+ (".qkv_proj", ".k_proj", "k"),
173
+ (".qkv_proj", ".v_proj", "v"),
174
+ (".gate_up_proj", ".gate_proj", 0),
175
+ (".gate_up_proj", ".up_proj", 1),
176
+ ]
177
+ params_dict = dict(self.named_parameters())
178
+ loaded_params: set[str] = set()
179
+ for name, loaded_weight in weights:
180
+ if name == "lm_head.weight":
181
+ spec_layer = self.model.mtp_start_layer_idx
182
+ name = f"model.layers.{spec_layer}.shared_head.head.weight"
183
+ elif name == "model.embed_tokens.weight":
184
+ spec_layer = self.model.mtp_start_layer_idx
185
+ else:
186
+ spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
187
+ if spec_layer is None:
188
+ continue
189
+
190
+ name = self._rewrite_spec_layer_name(spec_layer, name)
191
+
192
+ if self.quant_config is not None and (
193
+ scale_name := self.quant_config.get_cache_scale(name)
194
+ ):
195
+ # Loading kv cache quantization scales
196
+ param = params_dict[scale_name]
197
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
198
+ loaded_weight = (
199
+ loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]
200
+ )
201
+ weight_loader(param, loaded_weight)
202
+ loaded_params.add(scale_name)
203
+ continue
204
+
205
+ if "scale" in name or "zero_point" in name:
206
+ # Remapping the name of FP8 kv-scale or zero point.
207
+ name = maybe_remap_kv_scale_name(name, params_dict)
208
+ if name is None:
209
+ continue
210
+
211
+ for param_name, weight_name, shard_id in stacked_params_mapping:
212
+ if weight_name not in name:
213
+ continue
214
+ name = name.replace(weight_name, param_name)
215
+ # Skip loading extra bias for GPTQ models.
216
+ if name.endswith(".bias") and name not in params_dict:
217
+ continue
218
+
219
+ if is_pp_missing_parameter(name, self):
220
+ continue
221
+
222
+ param = params_dict[name]
223
+ weight_loader = param.weight_loader
224
+ weight_loader(param, loaded_weight, shard_id)
225
+ break
226
+ else:
227
+ # Skip loading extra bias for GPTQ models.
228
+ if name.endswith(".bias") and name not in params_dict:
229
+ continue
230
+ # Some checkpoints include weight scale tensors for the
231
+ # LM head even when the quantized head isn't built. Skip
232
+ # them if the model does not expose a matching parameter
233
+ # to avoid KeyError during load.
234
+ if name.endswith(".weight_scale") and name not in params_dict:
235
+ continue
236
+
237
+ # According to DeepSeek-V3 Technical Report, MTP modules
238
+ # shares embedding layer. We only load the first weights.
239
+ if (
240
+ spec_layer != self.model.mtp_start_layer_idx
241
+ and ".layers" not in name
242
+ ):
243
+ continue
244
+
245
+ if is_pp_missing_parameter(name, self):
246
+ continue
247
+ param = params_dict[name]
248
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
249
+ weight_loader(param, loaded_weight)
250
+ loaded_params.add(name)
251
+ return loaded_params
252
+
253
+ def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
254
+ """
255
+ Rewrite the weight name to match the format of the original model.
256
+ Add .mtp_block for modules in transformer layer block for spec layer
257
+ and rename shared layer weights to be top level.
258
+ """
259
+ name = name.replace("model.language_model.layers", "model.layers")
260
+
261
+ spec_layer_weight_names = [
262
+ "embed_tokens",
263
+ "enorm",
264
+ "hnorm",
265
+ "eh_proj",
266
+ "shared_head",
267
+ ]
268
+ shared_weight_names = ["embed_tokens"]
269
+ spec_layer_weight = False
270
+ shared_weight = False
271
+ for weight_name in spec_layer_weight_names:
272
+ if weight_name in name:
273
+ spec_layer_weight = True
274
+ if weight_name in shared_weight_names:
275
+ shared_weight = True
276
+ break
277
+ if not spec_layer_weight:
278
+ # treat rest weights as weights for transformer layer block
279
+ name = name.replace(
280
+ f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block."
281
+ )
282
+ elif shared_weight:
283
+ # treat shared weights as top level weights
284
+ name = name.replace(f"model.layers.{spec_layer}.", "model.")
285
+ return name
vllm/model_executor/models/registry.py
+4
-2
@@ -319,8 +319,9 @@ _MULTIMODAL_MODELS = {
319
),
320
"GlmAsrForConditionalGeneration": ("glmasr", "GlmAsrForConditionalGeneration"),
321
"GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
322
- "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"), # noqa: E501
323
- "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"), # noqa: E501
322
+ "Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"),
323
+ "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"),
324
+ "GlmOcrForConditionalGeneration": ("glm_ocr", "GlmOcrForConditionalGeneration"), # noqa: E501
325
"GraniteSpeechForConditionalGeneration": (
326
"granite_speech",
327
"GraniteSpeechForConditionalGeneration",
@@ -472,6 +473,7 @@ _SPECULATIVE_DECODING_MODELS = {
473
"LongCatFlashMTPModel": ("longcat_flash_mtp", "LongCatFlashMTP"),
474
"Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
475
"Glm4MoeLiteMTPModel": ("glm4_moe_lite_mtp", "Glm4MoeLiteMTP"),
476
+ "GlmOcrMTPModel": ("glm_ocr_mtp", "GlmOcrMTP"),
477
"MedusaModel": ("medusa", "Medusa"),
478
"OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
479
"Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
vllm/transformers_utils/model_arch_config_convertor.py
+1
@@ -398,6 +398,7 @@ MODEL_ARCH_CONFIG_CONVERTORS = {
398
"qwen3_next_mtp": Qwen3NextMTPModelArchConfigConvertor,
399
"mimo_mtp": MimoMTPModelArchConfigConvertor,
400
"glm4_moe_mtp": GLM4MoeMTPModelArchConfigConvertor,
401
+ "glm_ocr_mtp": GLM4MoeMTPModelArchConfigConvertor,
402
"ernie_mtp": ErnieMTPModelArchConfigConvertor,
403
"pangu_ultra_moe_mtp": PanguUltraMoeMTPModelArchConfigConvertor,
404
"longcat_flash_mtp": LongCatFlashMTPModelArchConfigConvertor,
vllm/v1/spec_decode/eagle.py
+2
-1
@@ -403,7 +403,7 @@ class SpecDecodeBaseProposer:
403
return draft_token_ids.view(-1, 1)
404
405
if self.uses_mrope:
406
- positions = self.positions[:, last_token_indices]
406
+ positions = self.mrope_positions[:, last_token_indices]
407
else:
408
positions = self.positions[last_token_indices]
409
if self.method in (
@@ -1126,6 +1126,7 @@ class SpecDecodeBaseProposer:
1126
"Qwen2_5_VLForConditionalGeneration",
1127
"Qwen3VLForConditionalGeneration",
1128
"Qwen3VLMoeForConditionalGeneration",
1129
+ "GlmOcrForConditionalGeneration",
1130
]:
1131
self.model.config.image_token_index = target_model.config.image_token_id
1132
elif self.get_model_name(target_model) == "PixtralForConditionalGeneration":