@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5fc5ce0fe4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model] Added GLM-4 series hf format model support vllm==0.6.4 (#10561)

Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

sixgod committed Nov 28, 2024 at 22:53 UTC 5fc5ce0fe45f974fc8840175e8321652238400f0
5 files changed +30 -1
docs/source/models/supported_models.rst
+5
@@ -139,6 +139,11 @@ Text Generation
139 - :code:`google/gemma-2-9b`, :code:`google/gemma-2-27b`, etc.
140 - ✅︎
141 - ✅︎
142 + * - :code:`GlmForCausalLM`
143 + - GLM-4
144 + - :code:`THUDM/glm-4-9b-chat-hf`, etc.
145 + - ✅︎
146 + - ✅︎
147 * - :code:`GPT2LMHeadModel`
148 - GPT-2
149 - :code:`gpt2`, :code:`gpt2-xl`, etc.
tests/models/registry.py
+1
@@ -63,6 +63,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
63 "FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
64 "GemmaForCausalLM": _HfExamplesInfo("google/gemma-2b"),
65 "Gemma2ForCausalLM": _HfExamplesInfo("google/gemma-2-9b"),
66 + "GlmForCausalLM": _HfExamplesInfo("THUDM/glm-4-9b-chat-hf"),
67 "GPT2LMHeadModel": _HfExamplesInfo("gpt2"),
68 "GPTBigCodeForCausalLM": _HfExamplesInfo("bigcode/starcoder"),
69 "GPTJForCausalLM": _HfExamplesInfo("EleutherAI/gpt-j-6b"),
tests/models/test_initialization.py
+1 -1
@@ -11,7 +11,7 @@ from .registry import HF_EXAMPLE_MODELS
11
12 @pytest.mark.parametrize("model_arch", HF_EXAMPLE_MODELS.get_supported_archs())
13 def test_can_initialize(model_arch):
14 - if (model_arch == "Idefics3ForConditionalGeneration"
14 + if (model_arch in {"Idefics3ForConditionalGeneration", "GlmForCausalLM"}
15 and transformers.__version__ < "4.46.0"):
16 pytest.skip(reason="Model introduced in HF >= 4.46.0")
17
vllm/model_executor/models/glm.py new
+21
@@ -0,0 +1,21 @@
1 +"""Inference-only HF format GLM-4 model compatible with THUDM weights."""
2 +from vllm.config import VllmConfig
3 +from vllm.model_executor.models.llama import LlamaForCausalLM
4 +
5 +from .utils import PPMissingLayer
6 +
7 +
8 +class GlmForCausalLM(LlamaForCausalLM):
9 +
10 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
11 + super().__init__(vllm_config=vllm_config, prefix=prefix)
12 + # Hack Llama model to fit HF format GLM implementation
13 + # Attention difference between GLM and Llama:
14 + # 1. Half partial rotary_dim and no Neox style.
15 + # 2. There is no bias for o_proj in attention
16 + for layer in self.model.layers:
17 + if not isinstance(layer, PPMissingLayer):
18 + layer.self_attn.rotary_emb.rotary_dim //= 2
19 + layer.self_attn.rotary_emb.is_neox_style = False
20 + layer.self_attn.o_proj.bias = None
21 + layer.self_attn.o_proj.skip_bias_add = True
vllm/model_executor/models/registry.py
+2
@@ -48,6 +48,7 @@ _TEXT_GENERATION_MODELS = {
48 "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
49 "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
50 "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
51 + "GlmForCausalLM": ("glm", "GlmForCausalLM"),
52 "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
53 "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
54 "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
@@ -107,6 +108,7 @@ _EMBEDDING_MODELS = {
108 "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
109 "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
110 "Gemma2Model": ("gemma2", "Gemma2EmbeddingModel"),
111 + "GlmForCausalLM": ("glm", "GlmForCausalLM"),
112 "LlamaModel": ("llama", "LlamaEmbeddingModel"),
113 **{
114 # Multiple models share the same architecture, so we include them all