A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Enable Proper `attention_bias` Usage in Llama Model Configuration (#3767)
Co-authored-by: roy <jasonailu87@gmail.com>
Kiran R committed
Apr 9, 2024 at 01:12 UTC
bc0c0192d13ca6ea4aeea4725f752a89483895bc
1 file changed
+5
-1
vllm/model_executor/models/llama.py
+5
-1
@@ -184,6 +184,10 @@ class LlamaDecoderLayer(nn.Module):
184
max_position_embeddings = getattr(config, "max_position_embeddings",
185
8192)
186
sliding_window = getattr(config, "sliding_window", None)
187
+ # Support abacusai/Smaug-72B-v0.1 with attention_bias
188
+ # Support internlm/internlm-7b with bias
189
+ attention_bias = getattr(config, "attention_bias", False) or getattr(
190
+ config, "bias", False)
191
self.self_attn = LlamaAttention(
192
hidden_size=self.hidden_size,
193
num_heads=config.num_attention_heads,
@@ -193,7 +197,7 @@ class LlamaDecoderLayer(nn.Module):
197
rope_scaling=rope_scaling,
198
max_position_embeddings=max_position_embeddings,
199
linear_method=linear_method,
196
- bias=getattr(config, "bias", False),
200
+ bias=attention_bias,
201
sliding_window=sliding_window,
202
)
203
self.mlp = LlamaMLP(