@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4934d49274
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Support GPT-NeoX Models without attention biases (#2301)

Jong-hun Shin committed Dec 31, 2023 at 01:42 UTC 4934d492744d14104353b8236ef8a0405edf1622
1 file changed +3
vllm/model_executor/models/gpt_neox.py
+3
@@ -54,6 +54,7 @@ class GPTNeoXAttention(nn.Module):
54 self.total_num_heads = config.num_attention_heads
55 self.hidden_size = config.hidden_size
56 self.head_size = self.hidden_size // self.total_num_heads
57 + self.bias = getattr(config, "attention_bias", True)
58
59 tensor_model_parallel_world_size = (
60 get_tensor_model_parallel_world_size())
@@ -65,11 +66,13 @@ class GPTNeoXAttention(nn.Module):
66 config.hidden_size,
67 self.head_size,
68 self.total_num_heads,
69 + bias=self.bias,
70 linear_method=linear_method,
71 )
72 self.dense = RowParallelLinear(
73 config.hidden_size,
74 config.hidden_size,
75 + bias=self.bias,
76 linear_method=linear_method,
77 )
78 scaling = self.head_size**-0.5