A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Support GPT-NeoX Models without attention biases (#2301)
Jong-hun Shin committed
Dec 31, 2023 at 01:42 UTC
4934d492744d14104353b8236ef8a0405edf1622
1 file changed
+3
vllm/model_executor/models/gpt_neox.py
+3
@@ -54,6 +54,7 @@ class GPTNeoXAttention(nn.Module):
54
self.total_num_heads = config.num_attention_heads
55
self.hidden_size = config.hidden_size
56
self.head_size = self.hidden_size // self.total_num_heads
57
+ self.bias = getattr(config, "attention_bias", True)
58
59
tensor_model_parallel_world_size = (
60
get_tensor_model_parallel_world_size())
@@ -65,11 +66,13 @@ class GPTNeoXAttention(nn.Module):
66
config.hidden_size,
67
self.head_size,
68
self.total_num_heads,
69
+ bias=self.bias,
70
linear_method=linear_method,
71
)
72
self.dense = RowParallelLinear(
73
config.hidden_size,
74
config.hidden_size,
75
+ bias=self.bias,
76
linear_method=linear_method,
77
)
78
scaling = self.head_size**-0.5