A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Bugfix] Fix GLM4 model (#16618)
Signed-off-by: intervitens <intervitens@tutanota.com>
intervitens committed
Apr 17, 2025 at 13:35 UTC
5b1aca2ae39b4bcdd04d916ec55bfc87f98d4835
3 files changed
+6
-6
docs/source/models/supported_models.md
+1
-1
@@ -338,7 +338,7 @@ See [this page](#generative-models) for more information on how to use generativ
338
* ✅︎
339
- * `Glm4ForCausalLM`
340
* GLM-4-0414
341
- * `THUDM/GLM-4-32B-Chat-0414`, etc.
341
+ * `THUDM/GLM-4-32B-0414`, etc.
342
* ✅︎
343
* ✅︎
344
- * `GPT2LMHeadModel`
tests/models/registry.py
+1
-1
@@ -147,7 +147,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
147
min_transformers_version="4.50"),
148
"GlmForCausalLM": _HfExamplesInfo("THUDM/glm-4-9b-chat-hf"),
149
"Glm4ForCausalLM": _HfExamplesInfo(
150
- "THUDM/GLM-4-32B-Chat-0414",
150
+ "THUDM/GLM-4-32B-0414",
151
is_available_online=False,
152
min_transformers_version="4.52.dev0"
153
),
vllm/model_executor/models/glm4.py
+4
-4
@@ -82,7 +82,7 @@ class Glm4Attention(nn.Module):
82
partial_rotary_factor = getattr(config, "partial_rotary_factor", 0.5)
83
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
84
self.head_dim = head_dim or hidden_size // self.total_num_heads
85
- self.rotary_dim = int(partial_rotary_factor * self.head_dim)
85
+ self.rotary_dim = self.head_dim
86
self.q_size = self.num_heads * self.head_dim
87
self.kv_size = self.num_kv_heads * self.head_dim
88
self.scaling = self.head_dim**-0.5
@@ -110,6 +110,7 @@ class Glm4Attention(nn.Module):
110
base=self.rope_theta,
111
rope_scaling=rope_scaling,
112
partial_rotary_factor=partial_rotary_factor,
113
+ is_neox_style=False,
114
)
115
self.attn = Attention(self.num_heads,
116
self.head_dim,
@@ -197,13 +198,12 @@ class Glm4DecoderLayer(nn.Module):
198
)
199
200
hidden_states = self.post_self_attn_layernorm(hidden_states)
200
- hidden_states = residual + hidden_states
201
202
# Fully Connected
203
- hidden_states = self.post_attention_layernorm(hidden_states, residual)
203
+ hidden_states, residual = self.post_attention_layernorm(
204
+ hidden_states, residual)
205
hidden_states = self.mlp(hidden_states)
206
hidden_states = self.post_mlp_layernorm(hidden_states)
206
- hidden_states = residual + hidden_states
207
208
return hidden_states, residual
209