A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Enable GQA support in the prefix prefill kernels (#3007)
Signed-off-by: Tao He <sighingnow@gmail.com>
Tao He committed
Feb 27, 2024 at 17:14 UTC
71bcaf99e2cb2c677bf3a9addb9e8039cbcab22a
3 files changed
+87
-47
tests/kernels/test_prefix_prefill.py
+42
-19
@@ -8,7 +8,8 @@ from vllm.model_executor.layers.triton_kernel.prefix_prefill import (
8
from xformers import ops as xops
9
from xformers.ops.fmha.attn_bias import BlockDiagonalCausalFromBottomRightMask
10
11
-NUM_HEADS = [12]
11
+NUM_HEADS = [64]
12
+NUM_QUERIES_PER_KV = [1, 8, 64]
13
HEAD_SIZES = [128]
14
DTYPES = [torch.float16]
15
CUDA_DEVICES = [
@@ -17,12 +18,14 @@ CUDA_DEVICES = [
18
19
20
@pytest.mark.parametrize("num_heads", NUM_HEADS)
21
+@pytest.mark.parametrize("num_queries_per_kv", NUM_HEADS)
22
@pytest.mark.parametrize("head_size", HEAD_SIZES)
23
@pytest.mark.parametrize("dtype", DTYPES)
24
@pytest.mark.parametrize("device", CUDA_DEVICES)
25
@torch.inference_mode()
26
def test_contexted_kv_attention(
27
num_heads: int,
28
+ num_queries_per_kv: int,
29
head_size: int,
30
dtype: torch.dtype,
31
device: str,
@@ -41,28 +44,29 @@ def test_contexted_kv_attention(
44
subquery_lens = [random.randint(16, MAX_SEQ_LEN) for _ in range(BS)]
45
ctx_lens = [random.randint(16, MAX_CTX_LEN) for _ in range(BS)]
46
seq_lens = [a + b for a, b in zip(subquery_lens, ctx_lens)]
47
+ num_kv_heads = num_heads // num_queries_per_kv
48
49
num_tokens = sum(subquery_lens)
50
query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
51
query.uniform_(-1e-3, 1e-3)
52
output = torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
53
50
- kv = torch.empty(sum(seq_lens), 2, num_heads, head_size, dtype=dtype)
54
+ kv = torch.empty(sum(seq_lens), 2, num_kv_heads, head_size, dtype=dtype)
55
kv.uniform_(-1e-3, 1e-3)
56
key, value = kv.unbind(dim=1)
57
58
k_cache = torch.zeros(cache_size,
59
block_size,
56
- num_heads,
60
+ num_kv_heads,
61
head_size,
62
dtype=dtype)
63
v_cache = torch.zeros(cache_size,
64
block_size,
61
- num_heads,
65
+ num_kv_heads,
66
head_size,
67
dtype=dtype)
64
- k = torch.zeros(sum(subquery_lens), num_heads, head_size, dtype=dtype)
65
- v = torch.zeros(sum(subquery_lens), num_heads, head_size, dtype=dtype)
68
+ k = torch.zeros(sum(subquery_lens), num_kv_heads, head_size, dtype=dtype)
69
+ v = torch.zeros(sum(subquery_lens), num_kv_heads, head_size, dtype=dtype)
70
values = torch.arange(0, cache_size, dtype=torch.long)
71
values = values[torch.randperm(cache_size)]
72
block_table = values[:BS * max_block_per_request].view(
@@ -93,19 +97,21 @@ def test_contexted_kv_attention(
97
end_loc = start_loc + block_size
98
start_slot = block_table[i, block_id] * block_size
99
end_slot = start_slot + end_loc - start_loc
96
- k_cache.view(-1, num_heads, head_size)[start_slot:end_slot].copy_(
97
- key[start_loc:end_loc])
98
- v_cache.view(-1, num_heads, head_size)[start_slot:end_slot].copy_(
99
- value[start_loc:end_loc])
100
+ k_cache.view(-1, num_kv_heads,
101
+ head_size)[start_slot:end_slot].copy_(
102
+ key[start_loc:end_loc])
103
+ v_cache.view(-1, num_kv_heads,
104
+ head_size)[start_slot:end_slot].copy_(
105
+ value[start_loc:end_loc])
106
cur_ctx += block_size
107
block_id += 1
108
# transpose K_cache[num_blocks, block_size, num_kv_heads, head_size]
109
# to K_cache[num_blocks, num_kv_heads, head_size/8, block_size, 8]
104
- k_cache = k_cache.view(-1, block_size, num_heads, head_size // 8,
110
+ k_cache = k_cache.view(-1, block_size, num_kv_heads, head_size // 8,
111
8).permute(0, 2, 3, 1, 4).contiguous()
112
# transpose V_cache[num_blocks, block_size, num_kv_heads, head_size]
113
# to V_cache[num_blocks, num_kv_heads, head_size, block_size]
108
- v_cache = v_cache.view(-1, block_size, num_heads,
114
+ v_cache = v_cache.view(-1, block_size, num_kv_heads,
115
head_size).permute(0, 2, 3, 1).contiguous()
116
117
# Warm up the Triton kernel by calling it once before actually measuring generation time
@@ -123,12 +129,29 @@ def test_contexted_kv_attention(
129
130
attn_op = xops.fmha.cutlass.FwOp()
131
132
+ if num_kv_heads != num_heads:
133
+ # As of Nov 2023, xformers only supports MHA. For MQA/GQA,
134
+ # project the key and value tensors to the desired number of
135
+ # heads.
136
+ #
137
+ # see also: vllm/model_executor/layers/attention.py
138
+ query = query.view(query.shape[0], num_kv_heads, num_queries_per_kv,
139
+ query.shape[-1])
140
+ key = key[:, :, None, :].expand(key.shape[0], num_kv_heads,
141
+ num_queries_per_kv, key.shape[-1])
142
+ value = value[:, :,
143
+ None, :].expand(value.shape[0], num_kv_heads,
144
+ num_queries_per_kv, value.shape[-1])
145
+ query = query.unsqueeze(0)
146
+ key = key.unsqueeze(0)
147
+ value = value.unsqueeze(0)
148
+
149
attn_bias = BlockDiagonalCausalFromBottomRightMask.from_seqlens(
150
subquery_lens, seq_lens)
151
output_ref = xops.memory_efficient_attention_forward(
129
- query.unsqueeze(0),
130
- key.unsqueeze(0),
131
- value.unsqueeze(0),
152
+ query,
153
+ key,
154
+ value,
155
attn_bias=attn_bias,
156
p=0.0,
157
scale=scale,
@@ -137,9 +160,9 @@ def test_contexted_kv_attention(
160
torch.cuda.synchronize()
161
start_time = time.time()
162
output_ref = xops.memory_efficient_attention_forward(
140
- query.unsqueeze(0),
141
- key.unsqueeze(0),
142
- value.unsqueeze(0),
163
+ query,
164
+ key,
165
+ value,
166
attn_bias=attn_bias,
167
p=0.0,
168
scale=scale,
@@ -148,5 +171,5 @@ def test_contexted_kv_attention(
171
torch.cuda.synchronize()
172
end_time = time.time()
173
print(f"xformers Time: {(end_time - start_time)*1000:.2f} ms")
151
- output_ref = output_ref.squeeze(0)
174
+ output_ref = output_ref.squeeze(0, 2)
175
assert torch.allclose(output_ref, output, atol=1e-6, rtol=0)
vllm/model_executor/layers/attention.py
+18
-16
@@ -137,25 +137,27 @@ class PagedAttention(nn.Module):
137
)
138
139
if input_metadata.is_prompt:
140
- # Prompt run.
141
- if self.num_kv_heads != self.num_heads:
142
- # As of Nov 2023, xformers only supports MHA. For MQA/GQA,
143
- # project the key and value tensors to the desired number of
144
- # heads.
145
- # TODO(woosuk): Use MQA/GQA kernels for higher performance.
146
- query = query.view(query.shape[0], self.num_kv_heads,
147
- self.num_queries_per_kv, query.shape[-1])
148
- key = key[:, :,
149
- None, :].expand(key.shape[0], self.num_kv_heads,
150
- self.num_queries_per_kv,
151
- key.shape[-1])
152
- value = value[:, :, None, :].expand(value.shape[0],
153
- self.num_kv_heads,
154
- self.num_queries_per_kv,
155
- value.shape[-1])
140
# normal attention
141
if (key_cache is None or value_cache is None
142
or input_metadata.block_tables.numel() == 0):
143
+ if self.num_kv_heads != self.num_heads:
144
+ # As of Nov 2023, xformers only supports MHA. For MQA/GQA,
145
+ # project the key and value tensors to the desired number of
146
+ # heads.
147
+ # TODO(woosuk): Use MQA/GQA kernels for higher performance.
148
+ query = query.view(query.shape[0], self.num_kv_heads,
149
+ self.num_queries_per_kv,
150
+ query.shape[-1])
151
+ key = key[:, :,
152
+ None, :].expand(key.shape[0], self.num_kv_heads,
153
+ self.num_queries_per_kv,
154
+ key.shape[-1])
155
+ value = value[:, :,
156
+ None, :].expand(value.shape[0],
157
+ self.num_kv_heads,
158
+ self.num_queries_per_kv,
159
+ value.shape[-1])
160
+
161
# Set attention bias if not provided. This typically happens at
162
# the very attention layer of every iteration.
163
# FIXME(woosuk): This is a hack.
vllm/model_executor/layers/triton_kernel/prefix_prefill.py
+27
-12
@@ -45,6 +45,7 @@ if triton.__version__ >= "2.1.0":
45
stride_v_cache_h,
46
stride_v_cache_d,
47
stride_v_cache_bl,
48
+ num_queries_per_kv: int,
49
BLOCK_M: tl.constexpr,
50
BLOCK_DMODEL: tl.constexpr,
51
BLOCK_N: tl.constexpr,
@@ -53,6 +54,8 @@ if triton.__version__ >= "2.1.0":
54
cur_head = tl.program_id(1)
55
start_m = tl.program_id(2)
56
57
+ cur_kv_head = cur_head // num_queries_per_kv
58
+
59
cur_batch_ctx_len = tl.load(B_Ctxlen + cur_batch)
60
cur_batch_seq_len = tl.load(B_Seqlen + cur_batch)
61
cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch)
@@ -85,13 +88,14 @@ if triton.__version__ >= "2.1.0":
88
mask=(start_n + offs_n) < cur_batch_ctx_len,
89
other=0)
90
off_k = (bn[None, :] * stride_k_cache_bs +
88
- cur_head * stride_k_cache_h +
91
+ cur_kv_head * stride_k_cache_h +
92
(offs_d[:, None] // x) * stride_k_cache_d +
93
((start_n + offs_n[None, :]) % block_size) *
94
stride_k_cache_bl +
95
(offs_d[:, None] % x) * stride_k_cache_x)
96
off_v = (
94
- bn[:, None] * stride_v_cache_bs + cur_head * stride_v_cache_h +
97
+ bn[:, None] * stride_v_cache_bs +
98
+ cur_kv_head * stride_v_cache_h +
99
offs_d[None, :] * stride_v_cache_d +
100
(start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
101
k = tl.load(K_cache + off_k,
@@ -131,9 +135,9 @@ if triton.__version__ >= "2.1.0":
135
l_i = l_i_new
136
m_i = m_i_new
137
134
- off_k = (offs_n[None, :] * stride_kbs + cur_head * stride_kh +
138
+ off_k = (offs_n[None, :] * stride_kbs + cur_kv_head * stride_kh +
139
offs_d[:, None] * stride_kd)
136
- off_v = (offs_n[:, None] * stride_vbs + cur_head * stride_vh +
140
+ off_v = (offs_n[:, None] * stride_vbs + cur_kv_head * stride_vh +
141
offs_d[None, :] * stride_vd)
142
k_ptrs = K + off_k
143
v_ptrs = V + off_v
@@ -232,6 +236,7 @@ if triton.__version__ >= "2.1.0":
236
stride_v_cache_h,
237
stride_v_cache_d,
238
stride_v_cache_bl,
239
+ num_queries_per_kv: int,
240
BLOCK_M: tl.constexpr,
241
BLOCK_DMODEL: tl.constexpr,
242
BLOCK_N: tl.constexpr,
@@ -240,6 +245,8 @@ if triton.__version__ >= "2.1.0":
245
cur_head = tl.program_id(1)
246
start_m = tl.program_id(2)
247
248
+ cur_kv_head = cur_head // num_queries_per_kv
249
+
250
cur_batch_ctx_len = tl.load(B_Ctxlen + cur_batch)
251
cur_batch_seq_len = tl.load(B_Seqlen + cur_batch)
252
cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch)
@@ -272,13 +279,14 @@ if triton.__version__ >= "2.1.0":
279
mask=(start_n + offs_n) < cur_batch_ctx_len,
280
other=0)
281
off_k = (bn[None, :] * stride_k_cache_bs +
275
- cur_head * stride_k_cache_h +
282
+ cur_kv_head * stride_k_cache_h +
283
(offs_d[:, None] // x) * stride_k_cache_d +
284
((start_n + offs_n[None, :]) % block_size) *
285
stride_k_cache_bl +
286
(offs_d[:, None] % x) * stride_k_cache_x)
287
off_v = (
281
- bn[:, None] * stride_v_cache_bs + cur_head * stride_v_cache_h +
288
+ bn[:, None] * stride_v_cache_bs +
289
+ cur_kv_head * stride_v_cache_h +
290
offs_d[None, :] * stride_v_cache_d +
291
(start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
292
k = tl.load(K_cache + off_k,
@@ -317,9 +325,9 @@ if triton.__version__ >= "2.1.0":
325
l_i = l_i_new
326
m_i = m_i_new
327
320
- off_k = (offs_n[None, :] * stride_kbs + cur_head * stride_kh +
328
+ off_k = (offs_n[None, :] * stride_kbs + cur_kv_head * stride_kh +
329
offs_d[:, None] * stride_kd)
322
- off_v = (offs_n[:, None] * stride_vbs + cur_head * stride_vh +
330
+ off_v = (offs_n[:, None] * stride_vbs + cur_kv_head * stride_vh +
331
offs_d[None, :] * stride_vd)
332
k_ptrs = K + off_k
333
v_ptrs = V + off_v
@@ -420,6 +428,7 @@ if triton.__version__ >= "2.1.0":
428
stride_v_cache_h,
429
stride_v_cache_d,
430
stride_v_cache_bl,
431
+ num_queries_per_kv: int,
432
BLOCK_M: tl.constexpr,
433
BLOCK_DMODEL: tl.constexpr,
434
BLOCK_N: tl.constexpr,
@@ -429,6 +438,8 @@ if triton.__version__ >= "2.1.0":
438
cur_head = tl.program_id(1)
439
start_m = tl.program_id(2)
440
441
+ cur_kv_head = cur_head // num_queries_per_kv
442
+
443
# cur_batch_seq_len: the length of prompts
444
# cur_batch_ctx_len: the length of prefix
445
# cur_batch_in_all_start_index: the start id of the dim=0
@@ -468,13 +479,14 @@ if triton.__version__ >= "2.1.0":
479
mask=(start_n + offs_n) < cur_batch_ctx_len,
480
other=0)
481
off_k = (bn[None, :] * stride_k_cache_bs +
471
- cur_head * stride_k_cache_h +
482
+ cur_kv_head * stride_k_cache_h +
483
(offs_d[:, None] // x) * stride_k_cache_d +
484
((start_n + offs_n[None, :]) % block_size) *
485
stride_k_cache_bl +
486
(offs_d[:, None] % x) * stride_k_cache_x)
487
off_v = (
477
- bn[:, None] * stride_v_cache_bs + cur_head * stride_v_cache_h +
488
+ bn[:, None] * stride_v_cache_bs +
489
+ cur_kv_head * stride_v_cache_h +
490
offs_d[None, :] * stride_v_cache_d +
491
(start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
492
k = tl.load(K_cache + off_k,
@@ -522,9 +534,9 @@ if triton.__version__ >= "2.1.0":
534
l_i = l_i_new
535
m_i = m_i_new
536
525
- off_k = (offs_n[None, :] * stride_kbs + cur_head * stride_kh +
537
+ off_k = (offs_n[None, :] * stride_kbs + cur_kv_head * stride_kh +
538
offs_d[:, None] * stride_kd)
527
- off_v = (offs_n[:, None] * stride_vbs + cur_head * stride_vh +
539
+ off_v = (offs_n[:, None] * stride_vbs + cur_kv_head * stride_vh +
540
offs_d[None, :] * stride_vd)
541
k_ptrs = K + off_k
542
v_ptrs = V + off_v
@@ -628,6 +640,7 @@ if triton.__version__ >= "2.1.0":
640
641
sm_scale = 1.0 / (Lq**0.5)
642
batch, head = b_seq_len.shape[0], q.shape[1]
643
+ num_queries_per_kv = q.shape[1] // k.shape[1]
644
645
grid = (batch, head, triton.cdiv(max_input_len, BLOCK)) # batch, head,
646
@@ -674,6 +687,7 @@ if triton.__version__ >= "2.1.0":
687
v_cache.stride(2),
688
v_cache.stride(
689
3), #[num_blocks, num_kv_heads, head_size, block_size]
690
+ num_queries_per_kv=num_queries_per_kv,
691
BLOCK_M=BLOCK,
692
BLOCK_DMODEL=Lk,
693
BLOCK_N=BLOCK,
@@ -721,6 +735,7 @@ if triton.__version__ >= "2.1.0":
735
v_cache.stride(2),
736
v_cache.stride(
737
3), #[num_blocks, num_kv_heads, head_size, block_size]
738
+ num_queries_per_kv=num_queries_per_kv,
739
BLOCK_M=BLOCK,
740
BLOCK_DMODEL=Lk,
741
BLOCK_N=BLOCK,