@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3a0e1fc070
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Support for Stable LM 2 (#2598)

Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

dakotamahan-stability committed Jan 26, 2024 at 14:45 UTC 3a0e1fc070dc7482ab1c8fcdc961e5729a4cb0b3
1 file changed +2 -2
vllm/model_executor/models/stablelm.py
+2 -2
@@ -98,7 +98,7 @@ class StablelmAttention(nn.Module):
98 self.scaling = self.head_dim**-0.5
99 self.q_size = self.num_heads * self.head_dim
100 self.kv_size = self.num_key_value_heads * self.head_dim
101 -
101 + self.qkv_bias = getattr(config, "use_qkv_bias", False)
102 if (self.head_dim * self.num_heads * tp_size) != self.hidden_size:
103 raise ValueError(
104 f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
@@ -108,7 +108,7 @@ class StablelmAttention(nn.Module):
108 self.head_dim,
109 self.total_num_heads,
110 self.total_num_key_value_heads,
111 - bias=False,
111 + self.qkv_bias,
112 linear_method=linear_method)
113 self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,
114 self.hidden_size,