A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Support MPT with GQA (#1938)
Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Megha Agarwal committed
Dec 12, 2023 at 10:16 UTC
6428f1d051d731a9eca192950591e7a5a2788cb2
2 files changed
+28
-6
vllm/model_executor/layers/attention.py
+8
-4
@@ -138,7 +138,8 @@ class PagedAttention(nn.Module):
138
input_metadata.attn_bias = attn_bias
139
else:
140
input_metadata.attn_bias = _make_alibi_bias(
141
- self.alibi_slopes, batch_size, seq_len, query.dtype)
141
+ self.alibi_slopes, self.num_kv_heads, batch_size,
142
+ seq_len, query.dtype)
143
144
# TODO(woosuk): Too many view operations. Let's try to reduce them
145
# in the future for code readability.
@@ -180,31 +181,34 @@ class PagedAttention(nn.Module):
181
182
def _make_alibi_bias(
183
alibi_slopes: torch.Tensor,
184
+ num_kv_heads: int,
185
batch_size: int,
186
seq_len: int,
187
dtype: torch.dtype,
188
) -> LowerTriangularMaskWithTensorBias:
187
- bias = torch.arange(seq_len, dtype=dtype)
189
+ bias = torch.arange(seq_len, dtype=dtype, device="cuda")
190
# NOTE(zhuohan): HF uses
191
# `bias = bias[None, :].repeat(prompt_len, 1)`
192
# here. We find that both biases give the same results, but
193
# the bias below more accurately follows the original ALiBi
194
# paper.
195
bias = bias[None, :] - bias[:, None]
194
- bias = bias.to(alibi_slopes.device)
196
197
# When using custom attention bias, xformers requires the bias to
198
# be sliced from a tensor whose length is a multiple of 8.
199
padded_len = (seq_len + 7) // 8 * 8
200
+ num_heads = alibi_slopes.shape[0]
201
bias = torch.empty(
202
batch_size,
201
- alibi_slopes.shape[0],
203
+ num_heads,
204
seq_len,
205
padded_len,
206
device=alibi_slopes.device,
207
dtype=dtype,
208
)[:, :, :, :seq_len].copy_(bias)
209
bias.mul_(alibi_slopes[:, None, None])
210
+ if num_heads != num_kv_heads:
211
+ bias = bias.unflatten(1, (num_kv_heads, num_heads // num_kv_heads))
212
attn_bias = LowerTriangularMaskWithTensorBias(bias)
213
return attn_bias
214
vllm/model_executor/models/mpt.py
+20
-2
@@ -50,9 +50,14 @@ class MPTAttention(nn.Module):
50
super().__init__()
51
self.d_model = config.d_model
52
self.total_num_heads = config.n_heads
53
+ self.head_dim = self.d_model // self.total_num_heads
54
self.clip_qkv = config.attn_config["clip_qkv"]
55
self.qk_ln = config.attn_config["qk_ln"]
56
self.alibi_bias_max = config.attn_config["alibi_bias_max"]
57
+ if "kv_n_heads" in config.attn_config:
58
+ self.total_num_kv_heads = config.attn_config['kv_n_heads']
59
+ else:
60
+ self.total_num_kv_heads = self.total_num_heads
61
assert not config.attn_config["prefix_lm"]
62
assert config.attn_config["alibi"]
63
@@ -61,6 +66,7 @@ class MPTAttention(nn.Module):
66
self.d_model,
67
self.d_model // self.total_num_heads,
68
self.total_num_heads,
69
+ self.total_num_kv_heads,
70
bias=not config.no_bias,
71
linear_method=linear_method,
72
)
@@ -78,6 +84,17 @@ class MPTAttention(nn.Module):
84
assert self.total_num_heads % tp_world_size == 0
85
self.num_heads = self.total_num_heads // tp_world_size
86
87
+ if self.total_num_kv_heads >= tp_world_size:
88
+ # Number of KV heads is greater than TP size, so we partition
89
+ # the KV heads across multiple tensor parallel GPUs.
90
+ assert self.total_num_kv_heads % tp_world_size == 0
91
+ else:
92
+ # Number of KV heads is less than TP size, so we replicate
93
+ # the KV heads across multiple tensor parallel GPUs.
94
+ assert tp_world_size % self.total_num_kv_heads == 0
95
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size)
96
+ self.q_size = self.num_heads * self.head_dim
97
+ self.kv_size = self.num_kv_heads * self.head_dim
98
# Create the alibi slopes and slice them.
99
tp_rank = get_tensor_model_parallel_rank()
100
head_start = tp_rank * self.num_heads
@@ -91,7 +108,8 @@ class MPTAttention(nn.Module):
108
self.attn = PagedAttention(self.num_heads,
109
self.head_dim,
110
scaling,
94
- alibi_slopes=alibi_slopes)
111
+ alibi_slopes=alibi_slopes,
112
+ num_kv_heads=self.num_kv_heads)
113
114
def forward(
115
self,
@@ -105,7 +123,7 @@ class MPTAttention(nn.Module):
123
qkv, _ = self.Wqkv(hidden_states)
124
if self.clip_qkv is not None:
125
qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv)
108
- q, k, v = qkv.chunk(chunks=3, dim=-1)
126
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
127
if self.qk_ln:
128
q = self.q_ln(q)
129
k = self.k_ln(k)