A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Decouple glm4v (#22751)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Aug 13, 2025 at 08:13 UTC
fde0b611a37e442cb8a53999a1cce48d76f49c16
3 files changed
+23
-7
docs/models/supported_models.md
+1
-1
@@ -615,7 +615,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
615
| `Gemma3nForConditionalGeneration` | Gemma 3n | T + I + A | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | | ✅︎ |
616
| `GLM4VForCausalLM`<sup>^</sup> | GLM-4V | T + I | `zai-org/glm-4v-9b`, `zai-org/cogagent-9b-20241220`, etc. | ✅︎ | ✅︎ | ✅︎ |
617
| `Glm4vForConditionalGeneration` | GLM-4.1V-Thinking | T + I<sup>E+</sup> + V<sup>E+</sup> | `zai-org/GLM-4.1V-9B-Thinking`, etc. | ✅︎ | ✅︎ | ✅︎ |
618
-| `Glm4vMoeForConditionalGeneration` | GLM-4.5V | T + I<sup>E+</sup> + V<sup>E+</sup> | `zai-org/GLM-4.5V`, etc. | ✅︎ | ✅︎ | ✅︎ |
618
+| `Glm4vMoeForConditionalGeneration` | GLM-4.5V | T + I<sup>E+</sup> + V<sup>E+</sup> | `zai-org/GLM-4.5V`, etc. | | ✅︎ | ✅︎ |
619
| `GraniteSpeechForConditionalGeneration` | Granite Speech | T + A | `ibm-granite/granite-speech-3.3-8b` | ✅︎ | ✅︎ | ✅︎ |
620
| `H2OVLChatModel` | H2OVL | T + I<sup>E+</sup> | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | | ✅︎ | ✅︎ |
621
| `Idefics3ForConditionalGeneration` | Idefics3 | T + I | `HuggingFaceM4/Idefics3-8B-Llama3`, etc. | ✅︎ | | ✅︎ |
vllm/model_executor/models/glm4_1v.py
+21
-5
@@ -1227,10 +1227,7 @@ class Glm4vForConditionalGeneration(nn.Module, SupportsMultiModal,
1227
"k_proj",
1228
"v_proj",
1229
],
1230
- "gate_up_proj": [
1231
- "gate_proj",
1232
- "up_proj",
1233
- ],
1230
+ "gate_up_proj": ["gate_up_proj"]
1231
}
1232
1233
# To ensure correct weight loading and mapping.
@@ -1567,7 +1564,26 @@ class Glm4vForConditionalGeneration(nn.Module, SupportsMultiModal,
1564
Get the module prefix in multimodal models
1565
"""
1566
return MultiModelKeys.from_string_field(
1570
- language_model="language_model",
1567
+ language_model="language_model.model",
1568
connector="visual.merger.",
1569
tower_model="visual.",
1570
)
1571
+
1572
+
1573
+@MULTIMODAL_REGISTRY.register_processor(
1574
+ Glm4vMultiModalProcessor,
1575
+ info=Glm4vProcessingInfo,
1576
+ dummy_inputs=Glm4vDummyInputsBuilder,
1577
+)
1578
+class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration):
1579
+ packed_modules_mapping = {
1580
+ "qkv_proj": [
1581
+ "q_proj",
1582
+ "k_proj",
1583
+ "v_proj",
1584
+ ],
1585
+ "gate_up_proj": [
1586
+ "gate_proj",
1587
+ "up_proj",
1588
+ ],
1589
+ }
vllm/model_executor/models/registry.py
+1
-1
@@ -208,7 +208,7 @@ _MULTIMODAL_MODELS = {
208
"Gemma3nForConditionalGeneration": ("gemma3n_mm", "Gemma3nForConditionalGeneration"), # noqa: E501
209
"GLM4VForCausalLM": ("glm4v", "GLM4VForCausalLM"),
210
"Glm4vForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"), # noqa: E501
211
- "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vForConditionalGeneration"), # noqa: E501
211
+ "Glm4vMoeForConditionalGeneration": ("glm4_1v", "Glm4vMoeForConditionalGeneration"), # noqa: E501
212
"GraniteSpeechForConditionalGeneration": ("granite_speech", "GraniteSpeechForConditionalGeneration"), # noqa: E501
213
"H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
214
"InternVLChatModel": ("internvl", "InternVLChatModel"),