A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Model] Added GLM-4 series hf format model support vllm==0.6.4 (#10561)
Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
sixgod committed
Nov 28, 2024 at 22:53 UTC
5fc5ce0fe45f974fc8840175e8321652238400f0
5 files changed
+30
-1
docs/source/models/supported_models.rst
+5
@@ -139,6 +139,11 @@ Text Generation
139
- :code:`google/gemma-2-9b`, :code:`google/gemma-2-27b`, etc.
140
- ✅︎
141
- ✅︎
142
+ * - :code:`GlmForCausalLM`
143
+ - GLM-4
144
+ - :code:`THUDM/glm-4-9b-chat-hf`, etc.
145
+ - ✅︎
146
+ - ✅︎
147
* - :code:`GPT2LMHeadModel`
148
- GPT-2
149
- :code:`gpt2`, :code:`gpt2-xl`, etc.
tests/models/registry.py
+1
@@ -63,6 +63,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
63
"FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
64
"GemmaForCausalLM": _HfExamplesInfo("google/gemma-2b"),
65
"Gemma2ForCausalLM": _HfExamplesInfo("google/gemma-2-9b"),
66
+ "GlmForCausalLM": _HfExamplesInfo("THUDM/glm-4-9b-chat-hf"),
67
"GPT2LMHeadModel": _HfExamplesInfo("gpt2"),
68
"GPTBigCodeForCausalLM": _HfExamplesInfo("bigcode/starcoder"),
69
"GPTJForCausalLM": _HfExamplesInfo("EleutherAI/gpt-j-6b"),
tests/models/test_initialization.py
+1
-1
@@ -11,7 +11,7 @@ from .registry import HF_EXAMPLE_MODELS
11
12
@pytest.mark.parametrize("model_arch", HF_EXAMPLE_MODELS.get_supported_archs())
13
def test_can_initialize(model_arch):
14
- if (model_arch == "Idefics3ForConditionalGeneration"
14
+ if (model_arch in {"Idefics3ForConditionalGeneration", "GlmForCausalLM"}
15
and transformers.__version__ < "4.46.0"):
16
pytest.skip(reason="Model introduced in HF >= 4.46.0")
17
vllm/model_executor/models/glm.py
new
+21
@@ -0,0 +1,21 @@
1
+"""Inference-only HF format GLM-4 model compatible with THUDM weights."""
2
+from vllm.config import VllmConfig
3
+from vllm.model_executor.models.llama import LlamaForCausalLM
4
+
5
+from .utils import PPMissingLayer
6
+
7
+
8
+class GlmForCausalLM(LlamaForCausalLM):
9
+
10
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
11
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
12
+ # Hack Llama model to fit HF format GLM implementation
13
+ # Attention difference between GLM and Llama:
14
+ # 1. Half partial rotary_dim and no Neox style.
15
+ # 2. There is no bias for o_proj in attention
16
+ for layer in self.model.layers:
17
+ if not isinstance(layer, PPMissingLayer):
18
+ layer.self_attn.rotary_emb.rotary_dim //= 2
19
+ layer.self_attn.rotary_emb.is_neox_style = False
20
+ layer.self_attn.o_proj.bias = None
21
+ layer.self_attn.o_proj.skip_bias_add = True
vllm/model_executor/models/registry.py
+2
@@ -48,6 +48,7 @@ _TEXT_GENERATION_MODELS = {
48
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
49
"GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
50
"Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
51
+ "GlmForCausalLM": ("glm", "GlmForCausalLM"),
52
"GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
53
"GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
54
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
@@ -107,6 +108,7 @@ _EMBEDDING_MODELS = {
108
"XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
109
"DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
110
"Gemma2Model": ("gemma2", "Gemma2EmbeddingModel"),
111
+ "GlmForCausalLM": ("glm", "GlmForCausalLM"),
112
"LlamaModel": ("llama", "LlamaEmbeddingModel"),
113
**{
114
# Multiple models share the same architecture, so we include them all