@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4dd6416faf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Fix stablelm (#3038)

Roy committed Feb 27, 2024 at 10:31 UTC 4dd6416faf7cc3035ac3f5c8375eb27e6b0eee80
2 files changed +11 -6
vllm/model_executor/models/__init__.py
+1
@@ -43,6 +43,7 @@ _MODELS = {
43 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
44 "RWForCausalLM": ("falcon", "FalconForCausalLM"),
45 "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
46 + "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
47 }
48
49 # Models not supported by ROCm.
vllm/model_executor/models/stablelm.py
+10 -6
@@ -94,7 +94,9 @@ class StablelmAttention(nn.Module):
94 1, self.total_num_key_value_heads // tp_size)
95 self.head_dim = self.hidden_size // self.total_num_heads
96 self.max_position_embeddings = config.max_position_embeddings
97 - self.rotary_ndims = int(self.head_dim * self.config.rope_pct)
97 + rope_pct = getattr(config, "rope_pct",
98 + getattr(config, "partial_rotary_factor", 1))
99 + self.rotary_ndims = int(self.head_dim * rope_pct)
100 self.scaling = self.head_dim**-0.5
101 self.q_size = self.num_heads * self.head_dim
102 self.kv_size = self.num_key_value_heads * self.head_dim
@@ -114,7 +116,6 @@ class StablelmAttention(nn.Module):
116 self.hidden_size,
117 bias=False,
118 linear_method=linear_method)
117 - self.rotary_ndims = int(self.head_dim * self.config.rope_pct)
119 self.rotary_emb = get_rope(
120 self.head_dim,
121 rotary_dim=self.rotary_ndims,
@@ -152,10 +153,11 @@ class StablelmDecoderLayer(nn.Module):
153 super().__init__()
154 self.self_attn = StablelmAttention(config)
155 self.mlp = StablelmMLP(config, linear_method)
155 - self.input_layernorm = nn.LayerNorm(config.hidden_size,
156 - eps=config.norm_eps)
156 + norm_eps = getattr(config, "norm_eps",
157 + getattr(config, "layer_norm_eps", 1e-05))
158 + self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=norm_eps)
159 self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
158 - eps=config.norm_eps)
160 + eps=norm_eps)
161
162 def forward(
163 self,
@@ -199,7 +201,9 @@ class StableLMEpochModel(nn.Module):
201 StablelmDecoderLayer(config, linear_method)
202 for _ in range(config.num_hidden_layers)
203 ])
202 - self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps)
204 + norm_eps = getattr(config, "norm_eps",
205 + getattr(config, "layer_norm_eps", 1e-05))
206 + self.norm = nn.LayerNorm(config.hidden_size, eps=norm_eps)
207
208 def forward(
209 self,