A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Support for Stable LM 2 (#2598)
Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
dakotamahan-stability committed
Jan 26, 2024 at 14:45 UTC
3a0e1fc070dc7482ab1c8fcdc961e5729a4cb0b3
1 file changed
+2
-2
vllm/model_executor/models/stablelm.py
+2
-2
@@ -98,7 +98,7 @@ class StablelmAttention(nn.Module):
98
self.scaling = self.head_dim**-0.5
99
self.q_size = self.num_heads * self.head_dim
100
self.kv_size = self.num_key_value_heads * self.head_dim
101
-
101
+ self.qkv_bias = getattr(config, "use_qkv_bias", False)
102
if (self.head_dim * self.num_heads * tp_size) != self.hidden_size:
103
raise ValueError(
104
f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
@@ -108,7 +108,7 @@ class StablelmAttention(nn.Module):
108
self.head_dim,
109
self.total_num_heads,
110
self.total_num_key_value_heads,
111
- bias=False,
111
+ self.qkv_bias,
112
linear_method=linear_method)
113
self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,
114
self.hidden_size,