A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix bias in InternLM (#1501)
Woosuk Kwon committed
Oct 30, 2023 at 00:24 UTC
aa9af07cac7b681d7195dbd5de621fc5a2acde99
1 file changed
+4
-2
vllm/model_executor/models/internlm.py
+4
-2
@@ -62,6 +62,7 @@ class InternLMAttention(nn.Module):
62
self,
63
hidden_size: int,
64
num_heads: int,
65
+ bias: bool,
66
rope_theta: float = 10000,
67
max_position_embeddings: int = 8192,
68
):
@@ -81,13 +82,13 @@ class InternLMAttention(nn.Module):
82
self.qkv_proj = ColumnParallelLinear(
83
hidden_size,
84
3 * self.total_num_heads * self.head_dim,
84
- bias=True,
85
+ bias=bias,
86
gather_output=False,
87
)
88
self.o_proj = RowParallelLinear(
89
self.total_num_heads * self.head_dim,
90
hidden_size,
90
- bias=True,
91
+ bias=bias,
92
input_is_parallel=True,
93
)
94
self.attn = PagedAttentionWithRoPE(
@@ -126,6 +127,7 @@ class InternLMDecoderLayer(nn.Module):
127
self.self_attn = InternLMAttention(
128
hidden_size=self.hidden_size,
129
num_heads=config.num_attention_heads,
130
+ bias=config.bias,
131
rope_theta=rope_theta,
132
max_position_embeddings=max_position_embeddings,
133
)