A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
self.gate dtype update for GLM-4.5 (#22203)
Signed-off-by: zRzRzRzRzRzRzR <2448370773@qq.com>
Yuxuan Zhang committed
Aug 5, 2025 at 10:12 UTC
6fa41e0c32f3f1b3d4f146c7f6a9872dcf9d0968
3 files changed
+4
-3
docs/models/supported_models.md
+1
-1
@@ -606,7 +606,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
606
| `GLM4VForCausalLM`<sup>^</sup> | GLM-4V | T + I | `zai-org/glm-4v-9b`, `zai-org/cogagent-9b-20241220`, etc. | ✅︎ | ✅︎ | ✅︎ |
607
| `Glm4vForConditionalGeneration` | GLM-4.1V-Thinking | T + I<sup>E+</sup> + V<sup>E+</sup> | `zai-org/GLM-4.1V-9B-Thinking`, etc. | ✅︎ | ✅︎ | ✅︎ |
608
| `Glm4MoeForCausalLM` | GLM-4.5 | T + I<sup>E+</sup> + V<sup>E+</sup> | `zai-org/GLM-4.5`, etc. | ✅︎ | ✅︎ | ✅︎ |
609
-| `Glm4v_moeForConditionalGeneration` | GLM-4.5V | T + I<sup>E+</sup> + V<sup>E+</sup> | `zai-org/GLM-4.5V-Air`, etc. | ✅︎ | ✅︎ | ✅︎ |
609
+| `Glm4v_moeForConditionalGeneration` | GLM-4.5V | T + I<sup>E+</sup> + V<sup>E+</sup> | `zai-org/GLM-4.5V`, etc. | ✅︎ | ✅︎ | ✅︎ |
610
| `GraniteSpeechForConditionalGeneration` | Granite Speech | T + A | `ibm-granite/granite-speech-3.3-8b` | ✅︎ | ✅︎ | ✅︎ |
611
| `H2OVLChatModel` | H2OVL | T + I<sup>E+</sup> | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | | ✅︎ | ✅︎ |
612
| `Idefics3ForConditionalGeneration` | Idefics3 | T + I | `HuggingFaceM4/Idefics3-8B-Llama3`, etc. | ✅︎ | | ✅︎ |
tests/models/registry.py
+1
-1
@@ -383,7 +383,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
383
trust_remote_code=True,
384
hf_overrides={"architectures": ["GLM4VForCausalLM"]}), # noqa: E501
385
"Glm4vForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.1V-9B-Thinking"), # noqa: E501
386
- "Glm4v_moeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V-Air",
386
+ "Glm4v_moeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V",
387
is_available_online=False), # noqa: E501
388
"H2OVLChatModel": _HfExamplesInfo("h2oai/h2ovl-mississippi-800m",
389
trust_remote_code=True,
vllm/model_executor/models/glm4_moe.py
+2
-1
@@ -123,6 +123,7 @@ class Glm4MoE(nn.Module):
123
config.n_routed_experts,
124
bias=False,
125
quant_config=None,
126
+ params_dtype=torch.float32,
127
prefix=f"{prefix}.gate")
128
129
self.gate.e_score_correction_bias = nn.Parameter(
@@ -180,7 +181,7 @@ class Glm4MoE(nn.Module):
181
182
if self.n_shared_experts is not None:
183
shared_output = self.shared_experts(hidden_states)
183
- router_logits, _ = self.gate(hidden_states)
184
+ router_logits, _ = self.gate(hidden_states.to(dtype=torch.float32))
185
final_hidden_states = self.experts(
186
hidden_states=hidden_states,
187
router_logits=router_logits) * self.routed_scaling_factor