@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bc0c0192d1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Enable Proper `attention_bias` Usage in Llama Model Configuration (#3767)

Co-authored-by: roy <jasonailu87@gmail.com>

Kiran R committed Apr 9, 2024 at 01:12 UTC bc0c0192d13ca6ea4aeea4725f752a89483895bc
1 file changed +5 -1
vllm/model_executor/models/llama.py
+5 -1
@@ -184,6 +184,10 @@ class LlamaDecoderLayer(nn.Module):
184 max_position_embeddings = getattr(config, "max_position_embeddings",
185 8192)
186 sliding_window = getattr(config, "sliding_window", None)
187 + # Support abacusai/Smaug-72B-v0.1 with attention_bias
188 + # Support internlm/internlm-7b with bias
189 + attention_bias = getattr(config, "attention_bias", False) or getattr(
190 + config, "bias", False)
191 self.self_attn = LlamaAttention(
192 hidden_size=self.hidden_size,
193 num_heads=config.num_attention_heads,
@@ -193,7 +197,7 @@ class LlamaDecoderLayer(nn.Module):
197 rope_scaling=rope_scaling,
198 max_position_embeddings=max_position_embeddings,
199 linear_method=linear_method,
196 - bias=getattr(config, "bias", False),
200 + bias=attention_bias,
201 sliding_window=sliding_window,
202 )
203 self.mlp = LlamaMLP(