@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aa9af07cac
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix bias in InternLM (#1501)

Woosuk Kwon committed Oct 30, 2023 at 00:24 UTC aa9af07cac7b681d7195dbd5de621fc5a2acde99
1 file changed +4 -2
vllm/model_executor/models/internlm.py
+4 -2
@@ -62,6 +62,7 @@ class InternLMAttention(nn.Module):
62 self,
63 hidden_size: int,
64 num_heads: int,
65 + bias: bool,
66 rope_theta: float = 10000,
67 max_position_embeddings: int = 8192,
68 ):
@@ -81,13 +82,13 @@ class InternLMAttention(nn.Module):
82 self.qkv_proj = ColumnParallelLinear(
83 hidden_size,
84 3 * self.total_num_heads * self.head_dim,
84 - bias=True,
85 + bias=bias,
86 gather_output=False,
87 )
88 self.o_proj = RowParallelLinear(
89 self.total_num_heads * self.head_dim,
90 hidden_size,
90 - bias=True,
91 + bias=bias,
92 input_is_parallel=True,
93 )
94 self.attn = PagedAttentionWithRoPE(
@@ -126,6 +127,7 @@ class InternLMDecoderLayer(nn.Module):
127 self.self_attn = InternLMAttention(
128 hidden_size=self.hidden_size,
129 num_heads=config.num_attention_heads,
130 + bias=config.bias,
131 rope_theta=rope_theta,
132 max_position_embeddings=max_position_embeddings,
133 )