A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 23:30 UTC
[bug-fix] GLM OCR Patch Merger context_dim (#37962)
Signed-off-by: JaredforReal <w13431838023@gmail.com>
Jared Wen committed
Mar 26, 2026 at 20:11 UTC
757eafcf37ba1db654bf9abb96e39054af0dc69d
2 files changed
+14
-4
vllm/model_executor/models/glm4_1v.py
+6
-1
@@ -38,7 +38,10 @@ import torch.nn as nn
38
import torch.nn.functional as F
39
from einops import rearrange
40
from transformers import BatchFeature, Glm4vProcessor
41
-from transformers.models.glm4v.configuration_glm4v import Glm4vVisionConfig
41
+from transformers.models.glm4v.configuration_glm4v import (
42
+ Glm4vTextConfig,
43
+ Glm4vVisionConfig,
44
+)
45
from transformers.models.glm4v.image_processing_glm4v import (
46
Glm4vImageProcessor,
47
smart_resize,
@@ -604,6 +607,7 @@ class Glm4vVisionEmbeddings(nn.Module):
607
class Glm4vVisionTransformer(nn.Module):
608
def __init__(
609
self,
610
+ text_config: Glm4vTextConfig,
611
vision_config: Glm4vVisionConfig,
612
norm_eps: float = 1e-6,
613
quant_config: QuantizationConfig | None = None,
@@ -1424,6 +1428,7 @@ class Glm4vForConditionalGeneration(
1428
1429
with self._mark_tower_model(vllm_config, {"image", "video"}):
1430
self.visual = Glm4vVisionTransformer(
1431
+ config.text_config,
1432
config.vision_config,
1433
norm_eps=getattr(config, "rms_norm_eps", 1e-5),
1434
quant_config=quant_config,
vllm/model_executor/models/glm_ocr.py
+8
-3
@@ -35,7 +35,10 @@ import torch.nn as nn
35
from einops import rearrange
36
37
if TYPE_CHECKING:
38
- from transformers.models.glm_ocr.configuration_glm_ocr import GlmOcrVisionConfig
38
+ from transformers.models.glm_ocr.configuration_glm_ocr import (
39
+ GlmOcrTextConfig,
40
+ GlmOcrVisionConfig,
41
+ )
42
43
from vllm.config import VllmConfig
44
from vllm.distributed import get_tensor_model_parallel_world_size, parallel_state
@@ -250,12 +253,13 @@ class GlmOcrPatchMerger(Glm4vPatchMerger):
253
class GlmOcrVisionTransformer(Glm4vVisionTransformer):
254
def __init__(
255
self,
256
+ text_config: "GlmOcrTextConfig",
257
vision_config: "GlmOcrVisionConfig",
258
norm_eps: float = 1e-5,
259
quant_config: QuantizationConfig | None = None,
260
prefix: str = "",
261
) -> None:
258
- super().__init__(vision_config, norm_eps, quant_config, prefix)
262
+ super().__init__(text_config, vision_config, norm_eps, quant_config, prefix)
263
264
del self.post_conv_layernorm
265
del self.embeddings
@@ -301,7 +305,7 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer):
305
)
306
self.merger = GlmOcrPatchMerger(
307
d_model=vision_config.out_hidden_size,
304
- context_dim=vision_config.out_hidden_size * vision_config.in_channels,
308
+ context_dim=text_config.intermediate_size,
309
quant_config=quant_config,
310
bias=False,
311
prefix=f"{prefix}.merger",
@@ -383,6 +387,7 @@ class GlmOcrForConditionalGeneration(Glm4vForConditionalGeneration):
387
388
with self._mark_tower_model(vllm_config, {"image", "video"}):
389
self.visual = GlmOcrVisionTransformer(
390
+ config.text_config,
391
config.vision_config,
392
norm_eps=getattr(config, "rms_norm_eps", 1e-5),
393
quant_config=quant_config,