A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[kernel] fix sliding window in prefix prefill Triton kernel (#4405)
Co-authored-by: SangBin Cho <rkooo567@gmail.com>
Michał Moskal committed
May 2, 2024 at 11:23 UTC
32881f3f3106e17d2fd52d8ac00217a0f0b2476a
6 files changed
+91
-23
tests/kernels/test_prefix_prefill.py
+30
-4
@@ -15,6 +15,7 @@ DTYPES = [torch.float16]
15
CUDA_DEVICES = [
16
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
17
]
18
+SLIDING_WINDOW = [0, 16, 64, 128, 256, 512, 2048]
19
20
21
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@@ -22,11 +23,13 @@ CUDA_DEVICES = [
23
@pytest.mark.parametrize("head_size", HEAD_SIZES)
24
@pytest.mark.parametrize("dtype", DTYPES)
25
@pytest.mark.parametrize("device", CUDA_DEVICES)
26
+@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
27
@torch.inference_mode()
28
def test_contexted_kv_attention(
29
num_heads: int,
30
num_queries_per_kv: int,
31
head_size: int,
32
+ sliding_window: int,
33
dtype: torch.dtype,
34
device: str,
35
) -> None:
@@ -123,12 +126,32 @@ def test_contexted_kv_attention(
126
127
# Warm up the Triton kernel by calling it once before actually measuring
128
# generation time
126
- context_attention_fwd(query, k, v, output, k_cache, v_cache, block_table,
127
- b_start_loc, b_seq_len, b_ctx_len, max_input_len)
129
+ context_attention_fwd(query,
130
+ k,
131
+ v,
132
+ output,
133
+ k_cache,
134
+ v_cache,
135
+ block_table,
136
+ b_start_loc,
137
+ b_seq_len,
138
+ b_ctx_len,
139
+ max_input_len,
140
+ sliding_window=sliding_window)
141
torch.cuda.synchronize()
142
start_time = time.time()
130
- context_attention_fwd(query, k, v, output, k_cache, v_cache, block_table,
131
- b_start_loc, b_seq_len, b_ctx_len, max_input_len)
143
+ context_attention_fwd(query,
144
+ k,
145
+ v,
146
+ output,
147
+ k_cache,
148
+ v_cache,
149
+ block_table,
150
+ b_start_loc,
151
+ b_seq_len,
152
+ b_ctx_len,
153
+ max_input_len,
154
+ sliding_window=sliding_window)
155
torch.cuda.synchronize()
156
end_time = time.time()
157
print(f"triton Time: {(end_time - start_time)*1000:.2f} ms")
@@ -156,6 +179,9 @@ def test_contexted_kv_attention(
179
180
attn_bias = BlockDiagonalCausalFromBottomRightMask.from_seqlens(
181
subquery_lens, seq_lens)
182
+ if sliding_window > 0:
183
+ attn_bias = attn_bias.make_local_attention_from_bottomright(
184
+ sliding_window)
185
output_ref = xops.memory_efficient_attention_forward(
186
query,
187
key,
vllm/attention/backends/flash_attn.py
+1
@@ -249,6 +249,7 @@ class FlashAttentionImpl(AttentionImpl):
249
prefill_meta.context_lens,
250
prefill_meta.max_subquery_len,
251
self.alibi_slopes,
252
+ self.sliding_window[0],
253
)
254
if decode_meta := attn_metadata.decode_metadata:
255
# Decoding run.
vllm/attention/backends/rocm_flash_attn.py
+1
@@ -307,6 +307,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
307
prefill_meta.context_lens,
308
prefill_meta.max_subquery_len,
309
self.alibi_slopes,
310
+ self.sliding_window[0],
311
)
312
313
if decode_meta := attn_metadata.decode_metadata:
vllm/attention/backends/xformers.py
+1
@@ -246,6 +246,7 @@ class XFormersImpl(AttentionImpl):
246
prefill_meta.context_lens,
247
prefill_meta.max_subquery_len,
248
self.alibi_slopes,
249
+ self.sliding_window,
250
)
251
assert output[:num_prefill_tokens].shape == out.shape
252
output[:num_prefill_tokens] = out
vllm/attention/ops/paged_attn.py
+2
@@ -172,6 +172,7 @@ class PagedAttention:
172
context_lens: torch.Tensor,
173
max_subquery_len: int,
174
alibi_slopes: Optional[torch.Tensor],
175
+ sliding_window: Optional[int],
176
) -> torch.Tensor:
177
output = torch.empty_like(query)
178
context_attention_fwd(
@@ -188,6 +189,7 @@ class PagedAttention:
189
context_lens,
190
max_subquery_len,
191
alibi_slopes,
192
+ sliding_window,
193
)
194
return output
195
vllm/attention/ops/prefix_prefill.py
+56
-19
@@ -50,6 +50,7 @@ if triton.__version__ >= "2.1.0":
50
BLOCK_DMODEL: tl.constexpr, # head size
51
BLOCK_DMODEL_PADDED: tl.constexpr, # head size padded to a power of 2
52
BLOCK_N: tl.constexpr,
53
+ SLIDING_WINDOW: tl.constexpr,
54
):
55
cur_batch = tl.program_id(0)
56
cur_head = tl.program_id(1)
@@ -62,42 +63,53 @@ if triton.__version__ >= "2.1.0":
63
cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch)
64
cur_batch_query_len = cur_batch_seq_len - cur_batch_ctx_len
65
66
+ # start position inside of the query
67
+ # generally, N goes over kv, while M goes over query_len
68
block_start_loc = BLOCK_M * start_m
69
70
# initialize offsets
71
+ # [N]; starts at 0
72
offs_n = tl.arange(0, BLOCK_N)
73
+ # [D]; starts at 0
74
offs_d = tl.arange(0, BLOCK_DMODEL_PADDED)
75
+ # [M]; starts at current position in query
76
offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
77
+ # [M,D]
78
off_q = (
79
(cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs +
80
cur_head * stride_qh + offs_d[None, :] * stride_qd)
81
82
dim_mask = tl.where(
76
- tl.arange(0, BLOCK_DMODEL_PADDED) < BLOCK_DMODEL, 1, 0).to(tl.int1)
83
+ tl.arange(0, BLOCK_DMODEL_PADDED) < BLOCK_DMODEL, 1,
84
+ 0).to(tl.int1) # [D]
85
86
q = tl.load(Q + off_q,
87
mask=dim_mask[None, :] &
88
(offs_m[:, None] < cur_batch_query_len),
81
- other=0.0)
89
+ other=0.0) # [M,D]
90
83
- # # initialize pointer to m and l
84
- m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
85
- l_i = tl.zeros([BLOCK_M], dtype=tl.float32)
86
- acc = tl.zeros([BLOCK_M, BLOCK_DMODEL_PADDED], dtype=tl.float32)
91
+ # initialize pointer to m and l
92
+ m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") # [M]
93
+ l_i = tl.zeros([BLOCK_M], dtype=tl.float32) # [M]
94
+ acc = tl.zeros([BLOCK_M, BLOCK_DMODEL_PADDED],
95
+ dtype=tl.float32) # [M,D]
96
97
+ # compute query against context (no causal mask here)
98
for start_n in range(0, cur_batch_ctx_len, BLOCK_N):
99
start_n = tl.multiple_of(start_n, BLOCK_N)
100
# -- compute qk ----
101
bn = tl.load(B_Loc + cur_batch * stride_b_loc_b +
102
((start_n + offs_n) // block_size) * stride_b_loc_s,
103
mask=(start_n + offs_n) < cur_batch_ctx_len,
94
- other=0)
104
+ other=0) # [N]
105
+ # [D,N]
106
off_k = (bn[None, :] * stride_k_cache_bs +
107
cur_kv_head * stride_k_cache_h +
108
(offs_d[:, None] // x) * stride_k_cache_d +
109
((start_n + offs_n[None, :]) % block_size) *
110
stride_k_cache_bl +
111
(offs_d[:, None] % x) * stride_k_cache_x)
112
+ # [N,D]
113
off_v = (
114
bn[:, None] * stride_v_cache_bs +
115
cur_kv_head * stride_v_cache_h +
@@ -106,23 +118,39 @@ if triton.__version__ >= "2.1.0":
118
k = tl.load(K_cache + off_k,
119
mask=dim_mask[:, None] &
120
((start_n + offs_n[None, :]) < cur_batch_ctx_len),
109
- other=0.0)
121
+ other=0.0) # [D,N]
122
111
- qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
123
+ qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) # [M,N]
124
qk += tl.dot(q, k)
125
qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk,
126
float("-inf"))
127
qk *= sm_scale
128
+ if SLIDING_WINDOW > 0:
129
+ # (cur_batch_ctx_len + offs_m[:, None]) are the positions of
130
+ # Q entries in sequence
131
+ # (start_n + offs_n[None, :]) are the positions of
132
+ # KV entries in sequence
133
+ # So the condition makes sure each entry in Q only attends
134
+ # to KV entries not more than SLIDING_WINDOW away.
135
+ #
136
+ # We can't use -inf here, because the
137
+ # sliding window may lead to the entire row being masked.
138
+ # This then makes m_ij contain -inf, which causes NaNs in
139
+ # exp().
140
+ qk = tl.where((cur_batch_ctx_len + offs_m[:, None]) -
141
+ (start_n + offs_n[None, :]) < SLIDING_WINDOW, qk,
142
+ -10000)
143
144
# -- compute m_ij, p, l_ij
118
- m_ij = tl.max(qk, 1)
119
- p = tl.exp(qk - m_ij[:, None])
120
- l_ij = tl.sum(p, 1)
145
+ m_ij = tl.max(qk, 1) # [M]
146
+ p = tl.exp(qk - m_ij[:, None]) # [M,N]
147
+ l_ij = tl.sum(p, 1) # [M]
148
# -- update m_i and l_i
122
- m_i_new = tl.maximum(m_i, m_ij)
123
- alpha = tl.exp(m_i - m_i_new)
124
- beta = tl.exp(m_ij - m_i_new)
125
- l_i_new = alpha * l_i + beta * l_ij
149
+ m_i_new = tl.maximum(m_i, m_ij) # [M]
150
+ alpha = tl.exp(m_i - m_i_new) # [M]
151
+ beta = tl.exp(m_ij - m_i_new) # [M]
152
+ l_i_new = alpha * l_i + beta * l_ij # [M]
153
+
154
# -- update output accumulator --
155
# scale p
156
p_scale = beta / l_i_new
@@ -134,7 +162,7 @@ if triton.__version__ >= "2.1.0":
162
v = tl.load(V_cache + off_v,
163
mask=dim_mask[None, :] &
164
((start_n + offs_n[:, None]) < cur_batch_ctx_len),
137
- other=0.0)
165
+ other=0.0) # [N,D]
166
167
p = p.to(v.dtype)
168
acc += tl.dot(p, v)
@@ -149,8 +177,10 @@ if triton.__version__ >= "2.1.0":
177
k_ptrs = K + off_k
178
v_ptrs = V + off_v
179
180
+ # block_mask is 0 when we're already past the current query length
181
block_mask = tl.where(block_start_loc < cur_batch_query_len, 1, 0)
182
183
+ # compute query against itself (with causal mask)
184
for start_n in range(0, block_mask * (start_m + 1) * BLOCK_M, BLOCK_N):
185
start_n = tl.multiple_of(start_n, BLOCK_N)
186
# -- compute qk ----
@@ -163,8 +193,13 @@ if triton.__version__ >= "2.1.0":
193
qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
194
qk += tl.dot(q, k)
195
qk *= sm_scale
196
+ # apply causal mask
197
qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk,
198
float("-inf"))
199
+ if SLIDING_WINDOW > 0:
200
+ qk = tl.where(
201
+ offs_m[:, None] -
202
+ (start_n + offs_n[None, :]) < SLIDING_WINDOW, qk, -10000)
203
204
# -- compute m_ij, p, l_ij
205
m_ij = tl.max(qk, 1)
@@ -636,7 +671,8 @@ if triton.__version__ >= "2.1.0":
671
b_seq_len,
672
b_ctx_len,
673
max_input_len,
639
- alibi_slopes=None):
674
+ alibi_slopes=None,
675
+ sliding_window=None):
676
677
cap = torch.cuda.get_device_capability()
678
BLOCK = 128 if cap[0] >= 8 else 64
@@ -644,7 +680,7 @@ if triton.__version__ >= "2.1.0":
680
Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1]
681
assert Lq == Lk and Lk == Lv
682
# round up Lk to a power of 2 - this is required for Triton block size
647
- Lk_padded = 2**((Lk - 1).bit_length())
683
+ Lk_padded = triton.next_power_of_2(Lk)
684
685
sm_scale = 1.0 / (Lq**0.5)
686
batch, head = b_seq_len.shape[0], q.shape[1]
@@ -749,6 +785,7 @@ if triton.__version__ >= "2.1.0":
785
BLOCK_DMODEL=Lk,
786
BLOCK_DMODEL_PADDED=Lk_padded,
787
BLOCK_N=BLOCK,
788
+ SLIDING_WINDOW=sliding_window if sliding_window is not None else 0,
789
num_warps=num_warps,
790
num_stages=1,
791
)