A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Neuron][Kernel] Support Longer Sequences in NKI-based Flash PagedAttention and Improve Efficiency (#12921)
Signed-off-by: Lingfan Yu <lingfany@amazon.com>
Lingfan Yu committed
Feb 11, 2025 at 21:12 UTC
e92694b6fe264a85371317295bca6643508034ef
2 files changed
+154
-180
tests/neuron/test_prefix_prefill.py
+67
-51
@@ -1,6 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-import random
3
from typing import Optional
4
5
import pytest
@@ -171,12 +170,22 @@ def ref_context_attention(
170
return output
171
172
173
+@pytest.mark.parametrize(
174
+ "block_size, large_tile_size",
175
+ [
176
+ (32, 2048), # 64 blocks
177
+ (32, 4096), # 128 blocks
178
+ (32, 8192), # 256 blocks
179
+ (64, 8192), # 128 blocks
180
+ ],
181
+)
182
@pytest.mark.parametrize(
183
"num_heads,num_queries_per_kv,head_size,mixed_precision",
184
[
185
(4, 2, 8, False),
186
(4, 2, 8, True),
187
(32, 8, 64, True),
188
+ (16, 2, 128, True),
189
],
190
)
191
@torch.inference_mode()
@@ -184,6 +193,8 @@ def test_contexted_kv_attention(
193
num_heads: int,
194
num_queries_per_kv: int,
195
head_size: int,
196
+ block_size: int,
197
+ large_tile_size,
198
mixed_precision: bool,
199
) -> None:
200
import os
@@ -192,40 +203,46 @@ def test_contexted_kv_attention(
203
204
from vllm.attention.ops.nki_flash_attn import flash_attn_varlen_nkifunc
205
206
+ assert large_tile_size % block_size == 0
207
+
208
device = xm.xla_device()
209
197
- os.environ["NEURON_CC_FLAGS"] = (
198
- " --model-type=transformer -O1 "
199
- " --internal-hlo2tensorizer-options='--verify-hlo' ")
210
+ compiler_flags = [
211
+ "--model-type=transformer -O1",
212
+ "--internal-hlo2tensorizer-options='--verify-hlo'",
213
+ "--retry_failed_compilation",
214
+ ]
215
+ compiler_flags_str = " ".join(compiler_flags)
216
+ os.environ["NEURON_CC_FLAGS"] = compiler_flags_str
217
201
- random.seed(0)
218
torch.manual_seed(0)
219
torch.set_printoptions(sci_mode=False)
220
205
- min_ctx_len = 2
206
- max_ctx_len = 64
207
- min_query_len = 2
208
- max_query_len = 64
209
- prefill_batch_size = 2
210
- decode_batch_size = 6
221
+ min_ctx_len = 32
222
+ max_ctx_len = 1024
223
+ min_query_len = 16
224
+ max_query_len = 512
225
+ prefill_batch_size = 4
226
+ decode_batch_size = 12
227
batch_size = prefill_batch_size + decode_batch_size
212
- block_size = 32
228
max_model_len = (max_query_len + max_ctx_len) * 4
229
230
max_block_per_request = max_model_len // block_size
231
dtype = torch.float32
232
cache_size = (batch_size * max_block_per_request) + 2
218
- ctx_lens = [
219
- random.randint(min_ctx_len, max_ctx_len)
220
- for _ in range(prefill_batch_size)
221
- ] + [
222
- random.randint(min_ctx_len, max_ctx_len)
223
- for _ in range(decode_batch_size)
224
- ]
225
- query_lens = [
226
- random.randint(min_query_len, max_query_len)
227
- for _ in range(prefill_batch_size)
228
- ] + [1 for _ in range(decode_batch_size)]
233
+ prefill_ctx_lens = torch.randint(min_ctx_len,
234
+ max_ctx_len + 1, (prefill_batch_size, ),
235
+ dtype=torch.long).tolist()
236
+ decode_ctx_lens = torch.randint(min_ctx_len,
237
+ max_ctx_len + 1, (decode_batch_size, ),
238
+ dtype=torch.long).tolist()
239
+ ctx_lens = prefill_ctx_lens + decode_ctx_lens
240
+ query_lens = torch.randint(
241
+ min_query_len,
242
+ max_query_len + 1,
243
+ (prefill_batch_size, ),
244
+ dtype=torch.long,
245
+ ).tolist() + [1 for _ in range(decode_batch_size)]
246
seq_lens = [a + b for a, b in zip(query_lens, ctx_lens)]
247
num_kv_heads = num_heads // num_queries_per_kv
248
@@ -254,7 +271,6 @@ def test_contexted_kv_attention(
271
values = values[torch.randperm(cache_size)]
272
block_table = values[:batch_size * max_block_per_request].view(
273
batch_size, max_block_per_request)
257
- torch.tensor(seq_lens, dtype=torch.long)
274
b_ctx_len = torch.tensor(ctx_lens, dtype=torch.long)
275
b_start_loc = torch.cumsum(torch.tensor([0] + query_lens[:-1],
276
dtype=torch.long),
@@ -311,9 +327,7 @@ def test_contexted_kv_attention(
327
# build neuron program
328
return_debug_tensors = False
329
B_P_SIZE = 128
314
- LARGE_TILE_SZ = 2048
315
- max_num_queries = (
316
- (sum(query_lens) + block_size - 1) // block_size) * block_size
330
+ LARGE_TILE_SZ = large_tile_size
331
332
def get_active_block_tables(block_tables, query_lens, seq_lens, block_size,
333
num_blocks):
@@ -332,26 +346,28 @@ def test_contexted_kv_attention(
346
0,
347
)
348
335
- def shift_bit_length(x):
336
- return 1 << (x - 1).bit_length()
349
+ def ceil_div(a, b):
350
+ return (a + b - 1) // b
351
+
352
+ def pad_to_multiple(a, b):
353
+ return ceil_div(a, b) * b
354
+
355
+ def pad_to_next_power_of_2(a):
356
+ assert a > 0
357
+ return 2**int(a - 1).bit_length()
358
359
# calculate input shapes
339
- max_num_queries_shifted = shift_bit_length(max_num_queries)
340
- max_num_queries_factor = B_P_SIZE // max_num_queries_shifted
341
- max_num_queries_padded = max_num_queries_shifted * max_num_queries_factor
342
- assert (max_num_queries_padded == B_P_SIZE
343
- ), "invalid {max_num_queries_padded=}"
360
+ max_num_queries = pad_to_multiple(sum(query_lens), block_size)
361
+ max_num_queries = pad_to_next_power_of_2(max_num_queries)
362
head_size_padded = B_P_SIZE
363
+ assert head_size_padded >= head_size
364
context_lens = torch.tensor(seq_lens) - torch.tensor(query_lens)
346
- num_active_blocks_shifted = shift_bit_length(
347
- ((context_lens + block_size - 1) // block_size).sum().item())
348
- num_active_blocks_factor = (LARGE_TILE_SZ // block_size //
349
- num_active_blocks_shifted)
350
- num_active_blocks = num_active_blocks_shifted * num_active_blocks_factor
351
- assert (num_active_blocks *
352
- block_size) == LARGE_TILE_SZ, "invalid {num_active_blocks=}"
365
+ num_active_blocks = ceil_div(context_lens, block_size).sum().item()
366
+ num_active_blocks = pad_to_multiple(num_active_blocks,
367
+ LARGE_TILE_SZ // block_size)
368
context_kv_len = num_active_blocks * block_size
354
- assert context_kv_len == LARGE_TILE_SZ, f"invalid {context_kv_len=}"
369
+ assert (context_kv_len %
370
+ LARGE_TILE_SZ == 0), f"invalid context_kv_len={context_kv_len}"
371
372
# pad QKV tensors
373
pad_dims = (
@@ -360,7 +376,7 @@ def test_contexted_kv_attention(
376
0,
377
0,
378
0,
363
- max_num_queries_padded - query.shape[0],
379
+ max_num_queries - query.shape[0],
380
)
381
query = F.pad(query, pad_dims, "constant", 0)
382
k = F.pad(k, pad_dims, "constant", 0)
@@ -397,7 +413,7 @@ def test_contexted_kv_attention(
413
0,
414
context_kv_len - prior_mask.shape[1],
415
0,
400
- B_P_SIZE - prior_mask.shape[0],
416
+ max_num_queries - prior_mask.shape[0],
417
),
418
"constant",
419
0,
@@ -406,9 +422,9 @@ def test_contexted_kv_attention(
422
active_mask,
423
(
424
0,
409
- B_P_SIZE - active_mask.shape[1],
425
+ max_num_queries - active_mask.shape[1],
426
0,
411
- B_P_SIZE - active_mask.shape[0],
427
+ max_num_queries - active_mask.shape[0],
428
),
429
"constant",
430
0,
@@ -430,6 +446,8 @@ def test_contexted_kv_attention(
446
n_kv_head=num_kv_heads,
447
head_size=head_size,
448
mixed_precision=mixed_precision,
449
+ LARGE_TILE_SZ=LARGE_TILE_SZ,
450
+ return_debug_tensors=return_debug_tensors,
451
)
452
453
if return_debug_tensors:
@@ -439,17 +457,15 @@ def test_contexted_kv_attention(
457
output_nki = flash_attn_varlen_nkifunc(*input_args, **input_kwargs)
458
debug_tensors = []
459
442
- output_nki = torch.tensor(output_nki).cpu()
460
debug_tensors = [torch.tensor(dt).cpu() for dt in debug_tensors]
461
462
num_actual_tokens = sum(query_lens)
446
- print(f"{num_actual_tokens=}")
463
# - o: shape (bs, n_heads, seq_q, d) -> (bs, seq_q, n_heads, d)
448
- output_nki = output_nki.permute(
449
- 0, 2, 1, 3)[:, :, :, :head_size].cpu()[0, :num_actual_tokens, :, :]
464
+ output_nki = output_nki.cpu().permute(0, 2, 1, 3)[:, :, :, :head_size]
465
+ output_nki = output_nki[0, :num_actual_tokens, :, :]
466
output_ref_padded = F.pad(
467
output_ref,
452
- (0, 0, 0, 0, 0, 0, 0, max_num_queries_padded - output_ref.shape[0]),
468
+ (0, 0, 0, 0, 0, 0, 0, max_num_queries - output_ref.shape[0]),
469
"constant",
470
0,
471
)
vllm/attention/ops/nki_flash_attn.py
+87
-129
@@ -28,7 +28,6 @@ class FlashConfig:
28
def transpose_p_local(p_local_transposed,
29
p_local,
30
LARGE_TILE_SZ,
31
- forward_mask,
31
B_F_SIZE=512):
32
for i in nl.affine_range(LARGE_TILE_SZ // B_F_SIZE):
33
if nisa.get_nc_version() == nisa.nc_version.gen3:
@@ -46,13 +45,13 @@ def transpose_p_local(p_local_transposed,
45
46
if nisa.get_nc_version() == nisa.nc_version.gen3:
47
p_local_t_tmp[:, j_128_slice] = nisa.dma_transpose(
49
- p_local[:, i_j_128_slice], mask=forward_mask)
48
+ p_local[:, i_j_128_slice])
49
else:
50
p_local_t_tmp[:, j_128_slice] = nisa.nc_transpose(
52
- p_local[:, i_j_128_slice], mask=forward_mask)
51
+ p_local[:, i_j_128_slice])
52
53
p_local_transposed[:, nl.ds(i * B_F_SIZE, B_F_SIZE)] = nl.copy(
55
- p_local_t_tmp, dtype=p_local_transposed.dtype, mask=forward_mask)
54
+ p_local_t_tmp, dtype=p_local_transposed.dtype)
55
56
57
@nki.jit
@@ -60,36 +59,25 @@ def _flash_attention_core(
59
q_local_tile,
60
k,
61
v,
63
- q_h_per_k_h,
64
- seqlen_q,
65
- nheads,
62
o_buffer,
63
l_buffer,
64
m_buffer,
69
- batch_id,
70
- head_id,
71
- gqa_head_idx,
65
q_tile_idx,
73
- local_k_large_tile_idx,
66
kernel_dtype,
67
acc_type,
68
flash_config: FlashConfig,
77
- use_causal_mask=False,
78
- continuous_batching_mask=None,
69
+ use_causal_mask,
70
+ tile_mask,
71
initialize=False,
72
B_P_SIZE=128,
73
B_F_SIZE=512,
74
B_D_SIZE=128,
83
- dropout_p=0.0,
84
- dropout_p_tensor=None,
85
- seed_tensor=None,
86
- logit_bias_tile=None,
75
qk_res_buffer=None,
76
):
77
"""
78
The flash attention core function to calculate self attention between a tile
79
of q and a block of K and V.
92
- The q_local_tile has (B_P_SIZE, B_F_SIZE), which is loaded into the SBUF
80
+ The q_local_tile has (B_P_SIZE, B_F_SIZE), which is loaded into the SBUF
81
already. The block size of K and V
82
is defined in the seq_tile_size of the flash_config. The results are stored
83
in the following three buffers
@@ -99,24 +87,9 @@ def _flash_attention_core(
87
"""
88
LARGE_TILE_SZ = flash_config.seq_tile_size
89
num_k_tile_per_large_tile = LARGE_TILE_SZ // B_F_SIZE
102
- seqlen_k = k.shape[-1]
103
- seqlen_q // B_P_SIZE
104
- seqlen_k // B_F_SIZE
105
-
106
- # TODO : support logit_bias with continuous_batching_mask
107
- assert not use_causal_mask, "causal mask is not supported."
108
- assert (continuous_batching_mask
109
- is not None), "continuous_batching_mask input is required."
110
- if continuous_batching_mask is not None:
111
- assert (
112
- logit_bias_tile
113
- is None), "continuous_batching_mask does not support logit_bias!"
90
91
# mask are used to only apply computation to the lower half of the matrix,
92
# which reduce the arithmetic intensity by half
117
- forward_mask = (q_tile_idx * B_P_SIZE >= local_k_large_tile_idx *
118
- LARGE_TILE_SZ if use_causal_mask else None)
119
-
93
qk_res_buf = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
94
buffer=nl.sbuf,
95
dtype=acc_type)
@@ -125,20 +98,27 @@ def _flash_attention_core(
98
for k_i in nl.affine_range(num_k_tile_per_large_tile):
99
k_i_b_f_slice = nl.ds(k_i * B_F_SIZE, B_F_SIZE)
100
128
- qk_psum = nl.zeros((par_dim(B_P_SIZE), B_F_SIZE),
129
- dtype=np.float32,
130
- buffer=nl.psum) # (128, 512)
131
- qk_psum[:, :] = nl.matmul(q_local_tile,
132
- k[:, k_i_b_f_slice],
133
- transpose_x=True,
134
- mask=None) # (p(128), 512)
135
-
136
- qk_res_buf[:, k_i_b_f_slice] = nl.where(
137
- continuous_batching_mask[:, k_i_b_f_slice],
138
- qk_psum[:, nl.ds(0, B_F_SIZE)],
139
- -9984.0,
140
- dtype=acc_type,
141
- )
101
+ if use_causal_mask:
102
+ multiplication_required_selection = (q_tile_idx * B_P_SIZE
103
+ >= k_i * B_F_SIZE)
104
+ else:
105
+ multiplication_required_selection = True
106
+
107
+ if multiplication_required_selection:
108
+ qk_psum = nl.ndarray((par_dim(B_P_SIZE), B_F_SIZE),
109
+ dtype=np.float32,
110
+ buffer=nl.psum) # (128, 512)
111
+ qk_psum[:, :] = nl.matmul(q_local_tile,
112
+ k[:, k_i_b_f_slice],
113
+ transpose_x=True) # (p(128), 512)
114
+ qk_res_buf[:, k_i_b_f_slice] = nl.where(
115
+ tile_mask[:, k_i_b_f_slice],
116
+ qk_psum[:, nl.ds(0, B_F_SIZE)],
117
+ -9984.0,
118
+ dtype=acc_type,
119
+ )
120
+ else:
121
+ qk_res_buf[:, k_i_b_f_slice] = -9984.0
122
123
# Calculate max of the current tile
124
max_local[:, k_i] = nisa.tensor_reduce(
@@ -147,7 +127,6 @@ def _flash_attention_core(
127
axis=(1, ),
128
dtype=acc_type,
129
negate=False,
150
- mask=forward_mask,
130
)
131
132
if qk_res_buffer is not None:
@@ -159,7 +138,6 @@ def _flash_attention_core(
138
axis=(1, ),
139
dtype=acc_type,
140
negate=False,
162
- mask=forward_mask,
141
)
142
143
o_previous_scaled = nl.ndarray((par_dim(B_P_SIZE), B_D_SIZE),
@@ -170,8 +148,7 @@ def _flash_attention_core(
148
m_current = max_
149
else:
150
m_previous = nl.copy(m_buffer[:, 0])
173
- m_buffer[:, 0] = nl.maximum(m_previous, max_,
174
- mask=forward_mask) # (128,1)
151
+ m_buffer[:, 0] = nl.maximum(m_previous, max_) # (128,1)
152
153
m_current = m_buffer[:, 0]
154
# Compute scaling factor
@@ -180,11 +157,8 @@ def _flash_attention_core(
157
m_previous,
158
bias=-1 * m_current,
159
scale=1.0,
183
- mask=forward_mask,
160
)
185
- o_previous_scaled[...] = nl.multiply(o_buffer[:, :],
186
- alpha,
187
- mask=forward_mask)
161
+ o_previous_scaled[...] = nl.multiply(o_buffer[:, :], alpha)
162
163
p_local = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
164
dtype=kernel_dtype)
@@ -207,10 +181,9 @@ def _flash_attention_core(
181
reduce_op=nl.add,
182
reduce_res=p_partial_sum[:, k_r_i],
183
dtype=kernel_dtype,
210
- mask=forward_mask,
184
)
185
213
- ps = nl.sum(p_partial_sum, axis=1, dtype=acc_type, mask=forward_mask)
186
+ ps = nl.sum(p_partial_sum, axis=1, dtype=acc_type)
187
188
p_local_transposed = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
189
dtype=kernel_dtype)
@@ -218,7 +191,6 @@ def _flash_attention_core(
191
p_local_transposed=p_local_transposed,
192
p_local=p_local,
193
LARGE_TILE_SZ=LARGE_TILE_SZ,
221
- forward_mask=forward_mask,
194
B_F_SIZE=B_F_SIZE,
195
)
196
@@ -230,27 +202,20 @@ def _flash_attention_core(
202
p_local_transposed[:, nl.ds(k_i * B_P_SIZE, B_P_SIZE)],
203
v[k_i, :, :],
204
transpose_x=True,
233
- mask=forward_mask,
205
) # (128, 128) (p(Br), d)
206
207
if initialize:
208
o_buffer[:, :] = nl.copy(pv_psum[:, :])
209
l_buffer[:, 0] = nl.add(nl.log(ps), max_)
210
else:
240
- o_buffer[:, :] = nl.add(o_previous_scaled, pv_psum, mask=forward_mask)
211
+ o_buffer[:, :] = nl.add(o_previous_scaled, pv_psum)
212
213
l_prev = l_buffer[:, 0]
214
l_exp = nl.add(
244
- nl.exp(
245
- nl.subtract(l_prev, m_current, mask=forward_mask),
246
- mask=forward_mask,
247
- ),
215
+ nl.exp(nl.subtract(l_prev, m_current)),
216
ps,
249
- mask=forward_mask,
217
)
251
- l_buffer[:, 0] = nl.add(m_current,
252
- nl.log(l_exp, mask=forward_mask),
253
- mask=forward_mask)
218
+ l_buffer[:, 0] = nl.add(m_current, nl.log(l_exp))
219
220
221
@nki.jit
@@ -279,6 +244,21 @@ def load_v_tile(v_hbm_tile, cur_v_tile, j, v_i, config):
244
)
245
246
247
+@nki.jit
248
+def load_block_tables(block_tables_hbm, num_tiles):
249
+ (num_blocks, ) = block_tables_hbm.shape
250
+ assert num_blocks % num_tiles == 0
251
+ num_blocks_per_tile = num_blocks // num_tiles
252
+ block_tables_hbm = block_tables_hbm.reshape(
253
+ (num_tiles, num_blocks_per_tile))
254
+ block_tables_buffer = nl.load(block_tables_hbm, dtype=nl.int32)
255
+ return block_tables_buffer
256
+
257
+
258
+def is_power_of_2(x):
259
+ return x > 0 and (x & (x - 1)) == 0
260
+
261
+
262
@nki.jit
263
def flash_paged_attention(
264
query,
@@ -316,24 +296,24 @@ def flash_paged_attention(
296
- We use paged cache blocks (key_cache, value_cache) to store KV cache.
297
298
IO tensor dtypes:
319
- - This kernel assumes all IO tensors have the same dtype except for
299
+ - This kernel assumes all IO tensors have the same dtype except for
300
block_tables (int32) and mask (int32)
321
- - If mixed_percision is True, then all Tensor Engine operation will be
322
- performed in bfloat16 and accumulation will be performed in float32.
301
+ - If mixed_percision is True, then all Tensor Engine operation will be
302
+ performed in bfloat16 and accumulation will be performed in float32.
303
Otherwise the intermediates will be in the same type as the inputs.
304
305
Compile-time Constants:
306
- softmax_scale: scaling for softmax, is None, default is `1.0/(d**0.5)`
307
- mixed_precision: flag to set non-matmul ops in fp32 precision, default
328
- is set to `true`, if false, we use same precision as input types
308
+ is set to `true`, if false, we use same precision as input types
309
- config: Instance of dataclass :class:`nki.kernels.attention.FlashConfig`
310
with Performance config parameters for flash attention with default
311
values
332
- seq_tile_size: `default=2048`, size of the kv tile size for attention
312
+ seq_tile_size: `default=2048`, size of the kv tile size for attention
313
computation reduction
314
315
GQA support Notes:
336
- the spmd kernel for launching kernel should be on kv_heads instead of
316
+ the spmd kernel for launching kernel should be on kv_heads instead of
317
nheads
318
319
Example usage:
@@ -415,18 +395,13 @@ def flash_paged_attention(
395
), f"Need B_P_SIZE ({B_P_SIZE}) to be divisible by {block_size=}"
396
num_large_k_tile = context_kv_len // LARGE_TILE_SZ
397
num_blocks_per_large_tile = LARGE_TILE_SZ // block_size
418
- assert (num_blocks_per_large_tile <= B_P_SIZE
419
- ), f"The number of blocks in each large tile " \
420
- f"({num_blocks_per_large_tile}) shouldn't exceed partition size {B_P_SIZE}"
421
-
422
- block_tables_sbuf = nl.full((par_dim(B_P_SIZE), num_large_k_tile),
423
- 0,
424
- dtype=np.int32,
425
- buffer=nl.sbuf)
426
- for j in nl.affine_range(num_large_k_tile):
427
- i_p = nl.arange(num_blocks_per_large_tile)[:, None]
428
- block_tables_sbuf[i_p, j] = nl.load(
429
- block_tables[j * num_blocks_per_large_tile + i_p], dtype=np.int32)
398
+ assert block_size % 32 == 0, "block_size is expected to be a multiple of 32"
399
+ assert is_power_of_2(
400
+ num_blocks_per_large_tile
401
+ ), "The number of blocks in each large tile is expected of be power of 2"
402
+ assert is_power_of_2(seqlen_q), "seqlen_q is expected to be power of 2"
403
+
404
+ block_tables_sbuf = load_block_tables(block_tables, num_large_k_tile)
405
406
# Global Flash Attention accumulators
407
o_buffer = nl.zeros(
@@ -457,7 +432,7 @@ def flash_paged_attention(
432
)
433
434
for k_i in nl.affine_range(num_blocks_per_large_tile):
460
- loaded = nl.load(key_cache[block_tables_sbuf[k_i, j], :,
435
+ loaded = nl.load(key_cache[block_tables_sbuf[j, k_i], :,
436
head_id, :])
437
cur_k_tile[:, nl.ds(k_i *
438
block_size, block_size)] = nl.transpose(loaded)
@@ -469,7 +444,7 @@ def flash_paged_attention(
444
num_blocks_per_partition):
445
v_i = (partition_idx * num_blocks_per_partition +
446
block_in_partition)
472
- loaded_v = nl.load(value_cache[block_tables_sbuf[v_i, j], :,
447
+ loaded_v = nl.load(value_cache[block_tables_sbuf[j, v_i], :,
448
head_id, :])
449
cur_v_tile[
450
partition_idx,
@@ -477,14 +452,15 @@ def flash_paged_attention(
452
:,
453
] = loaded_v
454
480
- cur_mask = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
481
- dtype=mask.dtype)
482
- for m_i in nl.affine_range(LARGE_TILE_SZ // B_F_SIZE):
483
- cur_mask[:, nl.ds(m_i * B_F_SIZE, B_F_SIZE)] = nl.load(
484
- mask[:, nl.ds(j * LARGE_TILE_SZ + m_i * B_F_SIZE, B_F_SIZE)])
485
-
486
- for i_q_h in nl.affine_range(q_h_per_k_h):
487
- for i in nl.affine_range(n_tile_q):
455
+ for i in nl.affine_range(n_tile_q):
456
+ cur_mask = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
457
+ dtype=mask.dtype)
458
+ for m_i in nl.affine_range(LARGE_TILE_SZ // B_F_SIZE):
459
+ cur_mask[:, nl.ds(m_i * B_F_SIZE, B_F_SIZE)] = nl.load(mask[
460
+ nl.ds(i * B_P_SIZE, B_P_SIZE),
461
+ nl.ds(j * LARGE_TILE_SZ + m_i * B_F_SIZE, B_F_SIZE),
462
+ ])
463
+ for i_q_h in nl.affine_range(q_h_per_k_h):
464
q_tile = nl.ndarray((B_D_SIZE, B_P_SIZE), dtype=kernel_dtype)
465
q_hbm_tile = query[batch_id, head_id * q_h_per_k_h + i_q_h]
466
q_sbuf_tile = nl.load(
@@ -497,35 +473,24 @@ def flash_paged_attention(
473
q_local_tile=q_tile,
474
k=cur_k_tile,
475
v=cur_v_tile,
500
- q_h_per_k_h=q_h_per_k_h,
501
- seqlen_q=seqlen_q,
502
- nheads=h,
476
o_buffer=o_buffer[i, i_q_h],
477
l_buffer=l_buffer[:, i, i_q_h],
478
m_buffer=m_buffer[i, i_q_h],
506
- batch_id=batch_id,
507
- head_id=head_id,
508
- gqa_head_idx=i_q_h,
479
q_tile_idx=i,
510
- local_k_large_tile_idx=j,
480
kernel_dtype=kernel_dtype,
481
acc_type=acc_type,
482
flash_config=config,
483
use_causal_mask=False,
515
- continuous_batching_mask=cur_mask,
484
+ tile_mask=cur_mask,
485
initialize=j == 0,
486
B_P_SIZE=B_P_SIZE,
487
B_F_SIZE=B_F_SIZE,
488
B_D_SIZE=B_D_SIZE,
520
- dropout_p=0.0,
521
- dropout_p_tensor=None,
522
- seed_tensor=None,
523
- logit_bias_tile=None,
489
)
490
491
# compute attention between input query, key and value
492
if key is not None and value is not None:
528
- B_F_SIZE = seqlen_q
493
+ B_F_SIZE = min(seqlen_q, B_F_SIZE)
494
LARGE_TILE_SZ = seqlen_q
495
active_config = FlashConfig(
496
seq_tile_size=LARGE_TILE_SZ,
@@ -552,11 +517,16 @@ def flash_paged_attention(
517
config=active_config,
518
)
519
555
- cur_mask = nl.ndarray((par_dim(B_P_SIZE), B_F_SIZE), dtype=mask.dtype)
556
- cur_mask[:, :] = nl.load(mask[:, nl.ds(context_kv_len, B_F_SIZE)])
520
+ for i in nl.affine_range(n_tile_q):
521
+ cur_mask = nl.load(
522
+ mask[
523
+ nl.ds(i * B_P_SIZE, B_P_SIZE),
524
+ nl.ds(context_kv_len, LARGE_TILE_SZ),
525
+ ],
526
+ dtype=mask.dtype,
527
+ )
528
+ for i_q_h in nl.affine_range(q_h_per_k_h):
529
558
- for i_q_h in nl.affine_range(q_h_per_k_h):
559
- for i in nl.affine_range(n_tile_q):
530
q_tile = nl.ndarray((B_D_SIZE, B_P_SIZE), dtype=kernel_dtype)
531
q_hbm_tile = query[batch_id, head_id * q_h_per_k_h + i_q_h]
532
q_sbuf_tile = nl.load(
@@ -568,32 +538,21 @@ def flash_paged_attention(
538
q_local_tile=q_tile,
539
k=cur_k_tile,
540
v=cur_v_tile,
571
- q_h_per_k_h=q_h_per_k_h,
572
- seqlen_q=seqlen_q,
573
- nheads=h,
541
o_buffer=o_buffer[i, i_q_h],
542
l_buffer=l_buffer[:, i, i_q_h],
543
m_buffer=m_buffer[i, i_q_h],
577
- batch_id=batch_id,
578
- head_id=head_id,
579
- gqa_head_idx=i_q_h,
544
q_tile_idx=i,
581
- local_k_large_tile_idx=0,
545
kernel_dtype=kernel_dtype,
546
acc_type=acc_type,
547
flash_config=active_config,
585
- use_causal_mask=False,
586
- continuous_batching_mask=cur_mask,
548
+ use_causal_mask=True,
549
+ tile_mask=cur_mask,
550
initialize=False,
551
B_P_SIZE=B_P_SIZE,
552
B_F_SIZE=B_F_SIZE,
553
B_D_SIZE=B_D_SIZE,
591
- dropout_p=0.0,
592
- dropout_p_tensor=None,
593
- seed_tensor=None,
594
- logit_bias_tile=None,
595
- qk_res_buffer=qk_res_buffer[i, i_q_h]
596
- if qk_res_buffer is not None else None,
554
+ qk_res_buffer=(qk_res_buffer[i, i_q_h]
555
+ if qk_res_buffer is not None else None),
556
)
557
558
# -- -- -- -- write output to buffer on HBM -- -- -- -- -- -- #
@@ -652,7 +611,6 @@ def flash_attn_varlen_nkifunc(
611
attn_mask,
612
n_kv_head=None,
613
head_size=None,
655
- B_P_SIZE=128,
614
LARGE_TILE_SZ=2048,
615
return_debug_tensors=False,
616
mixed_precision=True,