A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix][Kernel] allow non-power-of-two head sizes in prefix prefill (#4128)
Michał Moskal committed
Apr 18, 2024 at 00:51 UTC
e8cc7967ff8a6f8432747a9e87ab451d36e1ff57
2 files changed
+28
-18
tests/kernels/test_prefix_prefill.py
+1
-1
@@ -10,7 +10,7 @@ from vllm.attention.ops.prefix_prefill import context_attention_fwd
10
11
NUM_HEADS = [64]
12
NUM_QUERIES_PER_KV = [1, 8, 64]
13
-HEAD_SIZES = [128]
13
+HEAD_SIZES = [128, 96]
14
DTYPES = [torch.float16]
15
CUDA_DEVICES = [
16
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
vllm/attention/ops/prefix_prefill.py
+27
-17
@@ -47,7 +47,8 @@ if triton.__version__ >= "2.1.0":
47
stride_v_cache_bl,
48
num_queries_per_kv: int,
49
BLOCK_M: tl.constexpr,
50
- BLOCK_DMODEL: tl.constexpr,
50
+ BLOCK_DMODEL: tl.constexpr, # head size
51
+ BLOCK_DMODEL_PADDED: tl.constexpr, # head size padded to a power of 2
52
BLOCK_N: tl.constexpr,
53
):
54
cur_batch = tl.program_id(0)
@@ -59,26 +60,30 @@ if triton.__version__ >= "2.1.0":
60
cur_batch_ctx_len = tl.load(B_Ctxlen + cur_batch)
61
cur_batch_seq_len = tl.load(B_Seqlen + cur_batch)
62
cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch)
63
+ cur_batch_query_len = cur_batch_seq_len - cur_batch_ctx_len
64
65
block_start_loc = BLOCK_M * start_m
66
67
# initialize offsets
68
offs_n = tl.arange(0, BLOCK_N)
67
- offs_d = tl.arange(0, BLOCK_DMODEL)
69
+ offs_d = tl.arange(0, BLOCK_DMODEL_PADDED)
70
offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
71
off_q = (
72
(cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs +
73
cur_head * stride_qh + offs_d[None, :] * stride_qd)
74
73
- q = tl.load(
74
- Q + off_q,
75
- mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len,
76
- other=0.0)
75
+ dim_mask = tl.where(
76
+ tl.arange(0, BLOCK_DMODEL_PADDED) < BLOCK_DMODEL, 1, 0).to(tl.int1)
77
+
78
+ q = tl.load(Q + off_q,
79
+ mask=dim_mask[None, :] &
80
+ (offs_m[:, None] < cur_batch_query_len),
81
+ other=0.0)
82
83
# # initialize pointer to m and l
84
m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
85
l_i = tl.zeros([BLOCK_M], dtype=tl.float32)
81
- acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
86
+ acc = tl.zeros([BLOCK_M, BLOCK_DMODEL_PADDED], dtype=tl.float32)
87
88
for start_n in range(0, cur_batch_ctx_len, BLOCK_N):
89
start_n = tl.multiple_of(start_n, BLOCK_N)
@@ -99,7 +104,8 @@ if triton.__version__ >= "2.1.0":
104
offs_d[None, :] * stride_v_cache_d +
105
(start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
106
k = tl.load(K_cache + off_k,
102
- mask=(start_n + offs_n[None, :]) < cur_batch_ctx_len,
107
+ mask=dim_mask[:, None] &
108
+ ((start_n + offs_n[None, :]) < cur_batch_ctx_len),
109
other=0.0)
110
111
qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
@@ -126,7 +132,8 @@ if triton.__version__ >= "2.1.0":
132
acc = acc * acc_scale[:, None]
133
# update acc
134
v = tl.load(V_cache + off_v,
129
- mask=(start_n + offs_n[:, None]) < cur_batch_ctx_len,
135
+ mask=dim_mask[None, :] &
136
+ ((start_n + offs_n[:, None]) < cur_batch_ctx_len),
137
other=0.0)
138
139
p = p.to(v.dtype)
@@ -142,16 +149,15 @@ if triton.__version__ >= "2.1.0":
149
k_ptrs = K + off_k
150
v_ptrs = V + off_v
151
145
- block_mask = tl.where(
146
- block_start_loc < cur_batch_seq_len - cur_batch_ctx_len, 1, 0)
152
+ block_mask = tl.where(block_start_loc < cur_batch_query_len, 1, 0)
153
154
for start_n in range(0, block_mask * (start_m + 1) * BLOCK_M, BLOCK_N):
155
start_n = tl.multiple_of(start_n, BLOCK_N)
156
# -- compute qk ----
157
k = tl.load(k_ptrs +
158
(cur_batch_in_all_start_index + start_n) * stride_kbs,
153
- mask=(start_n + offs_n[None, :]) <
154
- cur_batch_seq_len - cur_batch_ctx_len,
159
+ mask=dim_mask[:, None] &
160
+ ((start_n + offs_n[None, :]) < cur_batch_query_len),
161
other=0.0)
162
163
qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
@@ -179,8 +185,8 @@ if triton.__version__ >= "2.1.0":
185
# update acc
186
v = tl.load(v_ptrs +
187
(cur_batch_in_all_start_index + start_n) * stride_vbs,
182
- mask=(start_n + offs_n[:, None]) <
183
- cur_batch_seq_len - cur_batch_ctx_len,
188
+ mask=dim_mask[None, :] &
189
+ ((start_n + offs_n[:, None]) < cur_batch_query_len),
190
other=0.0)
191
192
p = p.to(v.dtype)
@@ -195,7 +201,8 @@ if triton.__version__ >= "2.1.0":
201
out_ptrs = Out + off_o
202
tl.store(out_ptrs,
203
acc,
198
- mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len)
204
+ mask=dim_mask[None, :] &
205
+ (offs_m[:, None] < cur_batch_query_len))
206
return
207
208
@triton.jit
@@ -636,7 +643,8 @@ if triton.__version__ >= "2.1.0":
643
# shape constraints
644
Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1]
645
assert Lq == Lk and Lk == Lv
639
- assert Lk in {16, 32, 64, 128}
646
+ # round up Lk to a power of 2 - this is required for Triton block size
647
+ Lk_padded = 2**((Lk - 1).bit_length())
648
649
sm_scale = 1.0 / (Lq**0.5)
650
batch, head = b_seq_len.shape[0], q.shape[1]
@@ -646,6 +654,7 @@ if triton.__version__ >= "2.1.0":
654
655
num_warps = 8 if Lk <= 64 else 8
656
if alibi_slopes is not None:
657
+ assert Lk == Lk_padded
658
_fwd_kernel_alibi[grid](
659
q,
660
k,
@@ -738,6 +747,7 @@ if triton.__version__ >= "2.1.0":
747
num_queries_per_kv=num_queries_per_kv,
748
BLOCK_M=BLOCK,
749
BLOCK_DMODEL=Lk,
750
+ BLOCK_DMODEL_PADDED=Lk_padded,
751
BLOCK_N=BLOCK,
752
num_warps=num_warps,
753
num_stages=1,