A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Fix stablelm (#3038)
Roy committed
Feb 27, 2024 at 10:31 UTC
4dd6416faf7cc3035ac3f5c8375eb27e6b0eee80
2 files changed
+11
-6
vllm/model_executor/models/__init__.py
+1
@@ -43,6 +43,7 @@ _MODELS = {
43
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
44
"RWForCausalLM": ("falcon", "FalconForCausalLM"),
45
"StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
46
+ "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
47
}
48
49
# Models not supported by ROCm.
vllm/model_executor/models/stablelm.py
+10
-6
@@ -94,7 +94,9 @@ class StablelmAttention(nn.Module):
94
1, self.total_num_key_value_heads // tp_size)
95
self.head_dim = self.hidden_size // self.total_num_heads
96
self.max_position_embeddings = config.max_position_embeddings
97
- self.rotary_ndims = int(self.head_dim * self.config.rope_pct)
97
+ rope_pct = getattr(config, "rope_pct",
98
+ getattr(config, "partial_rotary_factor", 1))
99
+ self.rotary_ndims = int(self.head_dim * rope_pct)
100
self.scaling = self.head_dim**-0.5
101
self.q_size = self.num_heads * self.head_dim
102
self.kv_size = self.num_key_value_heads * self.head_dim
@@ -114,7 +116,6 @@ class StablelmAttention(nn.Module):
116
self.hidden_size,
117
bias=False,
118
linear_method=linear_method)
117
- self.rotary_ndims = int(self.head_dim * self.config.rope_pct)
119
self.rotary_emb = get_rope(
120
self.head_dim,
121
rotary_dim=self.rotary_ndims,
@@ -152,10 +153,11 @@ class StablelmDecoderLayer(nn.Module):
153
super().__init__()
154
self.self_attn = StablelmAttention(config)
155
self.mlp = StablelmMLP(config, linear_method)
155
- self.input_layernorm = nn.LayerNorm(config.hidden_size,
156
- eps=config.norm_eps)
156
+ norm_eps = getattr(config, "norm_eps",
157
+ getattr(config, "layer_norm_eps", 1e-05))
158
+ self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=norm_eps)
159
self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
158
- eps=config.norm_eps)
160
+ eps=norm_eps)
161
162
def forward(
163
self,
@@ -199,7 +201,9 @@ class StableLMEpochModel(nn.Module):
201
StablelmDecoderLayer(config, linear_method)
202
for _ in range(config.num_hidden_layers)
203
])
202
- self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps)
204
+ norm_eps = getattr(config, "norm_eps",
205
+ getattr(config, "layer_norm_eps", 1e-05))
206
+ self.norm = nn.LayerNorm(config.hidden_size, eps=norm_eps)
207
208
def forward(
209
self,