A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Neuron][Kernel] NKI-based flash-attention kernel with paged KV cache (#11277)
Signed-off-by: Liangfu Chen <liangfc@amazon.com> Co-authored-by: Jiangfei Duan <jfduan@outlook.com>
Liangfu Chen committed
Jan 27, 2025 at 17:31 UTC
ddee88d0ff2757bdef98a83a9c78af1ea4559758
3 files changed
+1126
-1
.buildkite/run-neuron-test.sh
+1
-1
@@ -54,4 +54,4 @@ docker run --rm -it --device=/dev/neuron0 --device=/dev/neuron1 --network host \
54
-e "NEURON_COMPILE_CACHE_URL=${NEURON_COMPILE_CACHE_MOUNT}" \
55
--name "${container_name}" \
56
${image_name} \
57
- /bin/bash -c "python3 /workspace/vllm/examples/offline_inference/neuron.py"
57
+ /bin/bash -c "python3 /workspace/vllm/examples/offline_inference/neuron.py && python3 -m pytest /workspace/vllm/tests/neuron/ -v --capture=tee-sys"
tests/neuron/test_prefix_prefill.py
new
+456
@@ -0,0 +1,456 @@
1
+import random
2
+from typing import Optional
3
+
4
+import pytest
5
+import torch
6
+import torch.nn.functional as F
7
+
8
+
9
+class BlockDiagonalCausalFromBottomRightMask:
10
+
11
+ @staticmethod
12
+ def _from_seqlens(query_lens, seq_lens, block_size=None):
13
+ from torch import logical_and, logical_or
14
+
15
+ contexted = block_size is None
16
+ context_lens = torch.tensor(seq_lens) - torch.tensor(query_lens)
17
+ n_queries = sum(query_lens)
18
+ num_seqs = len(query_lens)
19
+ if contexted:
20
+ key_lens_blockaligned = seq_lens
21
+ else:
22
+ n_blocks_per_seq = (context_lens + block_size - 1) // block_size
23
+ offset_per_seq = n_blocks_per_seq * block_size
24
+ key_lens_blockaligned = offset_per_seq[:num_seqs].tolist()
25
+ n_keys = sum(key_lens_blockaligned)
26
+
27
+ a = (torch.arange(n_queries).reshape(n_queries,
28
+ 1).expand(n_queries, n_keys))
29
+ b = torch.arange(n_keys).reshape(1, n_keys).expand(n_queries, n_keys)
30
+ q_cumsum = torch.tensor([0] + query_lens).cumsum(dim=0)
31
+ k_cumsum = torch.tensor([0] + key_lens_blockaligned).cumsum(dim=0)
32
+
33
+ prior_mask = torch.zeros(n_queries, n_keys)
34
+ new_masks: list[torch.Tensor] = []
35
+ for seq_id in range(num_seqs):
36
+ ri = q_cumsum[seq_id]
37
+ ci = k_cumsum[seq_id]
38
+ nr = query_lens[seq_id]
39
+
40
+ if contexted:
41
+ nc = seq_lens[seq_id]
42
+ a_offset = ci + nc - ri - nr
43
+ new_mask = (a + a_offset) >= b
44
+ else:
45
+ nc = context_lens[seq_id]
46
+ a_offset = ci + nc - 1
47
+ new_mask = a_offset >= b
48
+
49
+ left_mask = b >= ci
50
+ top_mask = a >= ri
51
+ bottom_mask = a < (ri + nr)
52
+
53
+ new_mask = logical_and(
54
+ logical_and(logical_and(new_mask, left_mask), top_mask),
55
+ bottom_mask,
56
+ )
57
+ prior_mask = logical_or(prior_mask, new_mask)
58
+ new_masks = new_masks + [new_mask]
59
+ return prior_mask
60
+
61
+ @staticmethod
62
+ def from_seqlens(query_lens, seq_lens, block_size=None):
63
+ contexted = block_size is None
64
+ if contexted:
65
+ prior_mask = BlockDiagonalCausalFromBottomRightMask._from_seqlens(
66
+ query_lens, seq_lens)
67
+ active_mask = None
68
+ else:
69
+ prior_mask = BlockDiagonalCausalFromBottomRightMask._from_seqlens(
70
+ query_lens, seq_lens, block_size)
71
+ active_mask = BlockDiagonalCausalFromBottomRightMask._from_seqlens(
72
+ query_lens, query_lens)
73
+ return prior_mask, active_mask
74
+
75
+
76
+def ref_softmax(x: torch.Tensor,
77
+ dim: int,
78
+ mixed_precision=False,
79
+ return_max_reduce=False):
80
+ max_value = torch.amax(x, dim=dim, keepdims=True)
81
+ exp = torch.exp(x - max_value)
82
+ if mixed_precision:
83
+ sum_value = torch.sum(exp.astype(torch.float32),
84
+ dim=dim,
85
+ keepdims=True).astype(x.dtype)
86
+ else:
87
+ sum_value = torch.sum(exp, dim=dim, keepdims=True)
88
+ if return_max_reduce:
89
+ return exp / sum_value, max_value, torch.reciprocal(sum_value)
90
+ return exp / sum_value
91
+
92
+
93
+def ref_masked_attention(
94
+ query: torch.Tensor,
95
+ key: torch.Tensor,
96
+ value: torch.Tensor,
97
+ scale: float,
98
+ attn_mask: Optional[torch.Tensor] = None,
99
+ return_max_reduce: Optional[bool] = False,
100
+) -> torch.Tensor:
101
+ scaled_qk = scale * torch.einsum("qhd,khd->hqk", query, key).float()
102
+ if attn_mask is not None:
103
+ masked_score = scaled_qk + attn_mask.float()
104
+ if return_max_reduce:
105
+ norm_score, cached_max, cached_sum_reciprocal = ref_softmax(
106
+ masked_score, dim=-1, return_max_reduce=True)
107
+ else:
108
+ norm_score = ref_softmax(masked_score, dim=-1)
109
+ out = torch.einsum("hqk,khd->qhd", norm_score, value)
110
+ if return_max_reduce:
111
+ return (
112
+ out,
113
+ cached_max,
114
+ cached_sum_reciprocal,
115
+ norm_score,
116
+ masked_score,
117
+ scaled_qk,
118
+ )
119
+ else:
120
+ return out
121
+
122
+
123
+def ref_context_attention(
124
+ query,
125
+ key,
126
+ value,
127
+ query_lens,
128
+ seq_lens,
129
+ head_size,
130
+ num_kv_heads,
131
+ num_heads,
132
+ num_queries_per_kv,
133
+ return_max_reduce=False,
134
+):
135
+ scale = float(1.0 / (head_size**0.5))
136
+ if num_queries_per_kv > 1:
137
+ # Handle MQA and GQA
138
+ key = torch.repeat_interleave(key, num_queries_per_kv, dim=1)
139
+ value = torch.repeat_interleave(value, num_queries_per_kv, dim=1)
140
+
141
+ attn_mask, _ = BlockDiagonalCausalFromBottomRightMask.from_seqlens(
142
+ query_lens, seq_lens)
143
+
144
+ # convert binary mask to -inf values
145
+ attn_mask = torch.logical_not(attn_mask)
146
+ attn_mask = attn_mask.float() * -30000
147
+
148
+ output, cached_max, cached_sum_reciprocal, lse, masked_score, scaled_qk = (
149
+ ref_masked_attention(
150
+ query,
151
+ key,
152
+ value,
153
+ scale,
154
+ attn_mask,
155
+ return_max_reduce=return_max_reduce,
156
+ ))
157
+
158
+ output = output.unsqueeze(1)
159
+ if return_max_reduce:
160
+ return (
161
+ output,
162
+ cached_max,
163
+ cached_sum_reciprocal,
164
+ lse,
165
+ masked_score,
166
+ scaled_qk,
167
+ )
168
+ else:
169
+ return output
170
+
171
+
172
+@pytest.mark.parametrize(
173
+ "num_heads,num_queries_per_kv,head_size,mixed_precision",
174
+ [
175
+ (4, 2, 8, False),
176
+ (4, 2, 8, True),
177
+ (32, 8, 64, True),
178
+ ],
179
+)
180
+@torch.inference_mode()
181
+def test_contexted_kv_attention(
182
+ num_heads: int,
183
+ num_queries_per_kv: int,
184
+ head_size: int,
185
+ mixed_precision: bool,
186
+) -> None:
187
+ import os
188
+
189
+ import torch_xla.core.xla_model as xm
190
+
191
+ from vllm.attention.ops.nki_flash_attn import flash_attn_varlen_nkifunc
192
+
193
+ device = xm.xla_device()
194
+
195
+ os.environ["NEURON_CC_FLAGS"] = (
196
+ " --model-type=transformer -O1 "
197
+ " --internal-hlo2tensorizer-options='--verify-hlo' ")
198
+
199
+ random.seed(0)
200
+ torch.manual_seed(0)
201
+ torch.set_printoptions(sci_mode=False)
202
+
203
+ min_ctx_len = 2
204
+ max_ctx_len = 64
205
+ min_query_len = 2
206
+ max_query_len = 64
207
+ prefill_batch_size = 2
208
+ decode_batch_size = 6
209
+ batch_size = prefill_batch_size + decode_batch_size
210
+ block_size = 32
211
+ max_model_len = (max_query_len + max_ctx_len) * 4
212
+
213
+ max_block_per_request = max_model_len // block_size
214
+ dtype = torch.float32
215
+ cache_size = (batch_size * max_block_per_request) + 2
216
+ ctx_lens = [
217
+ random.randint(min_ctx_len, max_ctx_len)
218
+ for _ in range(prefill_batch_size)
219
+ ] + [
220
+ random.randint(min_ctx_len, max_ctx_len)
221
+ for _ in range(decode_batch_size)
222
+ ]
223
+ query_lens = [
224
+ random.randint(min_query_len, max_query_len)
225
+ for _ in range(prefill_batch_size)
226
+ ] + [1 for _ in range(decode_batch_size)]
227
+ seq_lens = [a + b for a, b in zip(query_lens, ctx_lens)]
228
+ num_kv_heads = num_heads // num_queries_per_kv
229
+
230
+ num_tokens = sum(query_lens)
231
+ query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
232
+ query.uniform_(-1, 1)
233
+ torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
234
+
235
+ kv = torch.empty(sum(seq_lens), 2, num_kv_heads, head_size, dtype=dtype)
236
+ kv.uniform_(-1, 1)
237
+ key, value = kv.unbind(dim=1)
238
+
239
+ k_cache = torch.zeros(cache_size,
240
+ block_size,
241
+ num_kv_heads,
242
+ head_size,
243
+ dtype=dtype)
244
+ v_cache = torch.zeros(cache_size,
245
+ block_size,
246
+ num_kv_heads,
247
+ head_size,
248
+ dtype=dtype)
249
+ k = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
250
+ v = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
251
+ values = torch.arange(0, cache_size, dtype=torch.long)
252
+ values = values[torch.randperm(cache_size)]
253
+ block_table = values[:batch_size * max_block_per_request].view(
254
+ batch_size, max_block_per_request)
255
+ torch.tensor(seq_lens, dtype=torch.long)
256
+ b_ctx_len = torch.tensor(ctx_lens, dtype=torch.long)
257
+ b_start_loc = torch.cumsum(torch.tensor([0] + query_lens[:-1],
258
+ dtype=torch.long),
259
+ dim=0)
260
+ # copy kv to cache
261
+ b_seq_start_loc = torch.cumsum(torch.tensor([0] + seq_lens[:-1],
262
+ dtype=torch.long),
263
+ dim=0)
264
+ for i in range(batch_size):
265
+ for j in range(query_lens[i]):
266
+ k[b_start_loc[i] + j].copy_(key[b_seq_start_loc[i] + b_ctx_len[i] +
267
+ j])
268
+ v[b_start_loc[i] + j].copy_(value[b_seq_start_loc[i] +
269
+ b_ctx_len[i] + j])
270
+ cur_ctx = 0
271
+ block_id = 0
272
+ while cur_ctx < b_ctx_len[i]:
273
+ start_loc = b_seq_start_loc[i] + cur_ctx
274
+ if cur_ctx + block_size > b_ctx_len[i]:
275
+ end_loc = b_seq_start_loc[i] + b_ctx_len[i]
276
+ else:
277
+ end_loc = start_loc + block_size
278
+ start_slot = block_table[i, block_id] * block_size
279
+ end_slot = start_slot + end_loc - start_loc
280
+ k_cache.view(-1, num_kv_heads,
281
+ head_size)[start_slot:end_slot].copy_(
282
+ key[start_loc:end_loc])
283
+ v_cache.view(-1, num_kv_heads,
284
+ head_size)[start_slot:end_slot].copy_(
285
+ value[start_loc:end_loc])
286
+ cur_ctx += block_size
287
+ block_id += 1
288
+
289
+ (
290
+ output_ref,
291
+ cached_max,
292
+ cached_sum_reciprocal,
293
+ lse,
294
+ masked_score,
295
+ scaled_qk,
296
+ ) = ref_context_attention(
297
+ query,
298
+ key,
299
+ value,
300
+ query_lens,
301
+ seq_lens,
302
+ head_size,
303
+ num_kv_heads,
304
+ num_heads,
305
+ num_queries_per_kv,
306
+ return_max_reduce=True,
307
+ )
308
+
309
+ # build neuron program
310
+ return_debug_tensors = False
311
+ B_P_SIZE = 128
312
+ LARGE_TILE_SZ = 2048
313
+ max_num_queries = (
314
+ (sum(query_lens) + block_size - 1) // block_size) * block_size
315
+
316
+ def get_active_block_tables(block_tables, query_lens, seq_lens, block_size,
317
+ num_blocks):
318
+ context_lens = seq_lens - query_lens
319
+ blocks_per_seq = (context_lens + block_size - 1) // block_size
320
+ num_seqs = len(seq_lens)
321
+ active_blocks: list[int] = []
322
+ for seq_id in range(num_seqs):
323
+ active_blocks = (
324
+ active_blocks +
325
+ block_tables[seq_id, :blocks_per_seq[seq_id]].tolist())
326
+ return F.pad(
327
+ torch.tensor(active_blocks),
328
+ (0, num_blocks - len(active_blocks)),
329
+ "constant",
330
+ 0,
331
+ )
332
+
333
+ def shift_bit_length(x):
334
+ return 1 << (x - 1).bit_length()
335
+
336
+ # calculate input shapes
337
+ max_num_queries_shifted = shift_bit_length(max_num_queries)
338
+ max_num_queries_factor = B_P_SIZE // max_num_queries_shifted
339
+ max_num_queries_padded = max_num_queries_shifted * max_num_queries_factor
340
+ assert (max_num_queries_padded == B_P_SIZE
341
+ ), "invalid {max_num_queries_padded=}"
342
+ head_size_padded = B_P_SIZE
343
+ context_lens = torch.tensor(seq_lens) - torch.tensor(query_lens)
344
+ num_active_blocks_shifted = shift_bit_length(
345
+ ((context_lens + block_size - 1) // block_size).sum().item())
346
+ num_active_blocks_factor = (LARGE_TILE_SZ // block_size //
347
+ num_active_blocks_shifted)
348
+ num_active_blocks = num_active_blocks_shifted * num_active_blocks_factor
349
+ assert (num_active_blocks *
350
+ block_size) == LARGE_TILE_SZ, "invalid {num_active_blocks=}"
351
+ context_kv_len = num_active_blocks * block_size
352
+ assert context_kv_len == LARGE_TILE_SZ, f"invalid {context_kv_len=}"
353
+
354
+ # pad QKV tensors
355
+ pad_dims = (
356
+ 0,
357
+ head_size_padded - query.shape[2],
358
+ 0,
359
+ 0,
360
+ 0,
361
+ max_num_queries_padded - query.shape[0],
362
+ )
363
+ query = F.pad(query, pad_dims, "constant", 0)
364
+ k = F.pad(k, pad_dims, "constant", 0)
365
+ v = F.pad(v, pad_dims, "constant", 0)
366
+ k_cache = F.pad(k_cache, (0, head_size_padded - head_size), "constant", 0)
367
+ v_cache = F.pad(v_cache, (0, head_size_padded - head_size), "constant", 0)
368
+
369
+ # permute QKV tensors
370
+ # query: (1, n_heads, d, seq_q)
371
+ # key: (1, n_kv_heads, d, seq_k)
372
+ # value: (1, n_kv_heads, seq_v, d)
373
+ query = query.unsqueeze(0).permute(0, 2, 3, 1).contiguous()
374
+ k = k.unsqueeze(0).permute(0, 2, 3, 1).contiguous()
375
+ v = v.unsqueeze(0).permute(0, 2, 1, 3).contiguous()
376
+
377
+ # transform block table
378
+ active_block_table = get_active_block_tables(
379
+ block_table,
380
+ torch.tensor(query_lens),
381
+ torch.tensor(seq_lens),
382
+ block_size,
383
+ num_active_blocks,
384
+ )
385
+
386
+ # Build attention masks
387
+ prior_mask, active_mask = (
388
+ BlockDiagonalCausalFromBottomRightMask.from_seqlens(
389
+ query_lens, seq_lens, block_size=block_size))
390
+ attn_mask = torch.concat(
391
+ [
392
+ F.pad(
393
+ prior_mask,
394
+ (
395
+ 0,
396
+ context_kv_len - prior_mask.shape[1],
397
+ 0,
398
+ B_P_SIZE - prior_mask.shape[0],
399
+ ),
400
+ "constant",
401
+ 0,
402
+ ).bool(),
403
+ F.pad(
404
+ active_mask,
405
+ (
406
+ 0,
407
+ B_P_SIZE - active_mask.shape[1],
408
+ 0,
409
+ B_P_SIZE - active_mask.shape[0],
410
+ ),
411
+ "constant",
412
+ 0,
413
+ ).bool(),
414
+ ],
415
+ dim=1,
416
+ )
417
+
418
+ input_args = (
419
+ query.to(device=device),
420
+ k.to(device=device),
421
+ v.to(device=device),
422
+ k_cache.to(device=device),
423
+ v_cache.to(device=device),
424
+ active_block_table.to(torch.int32).to(device=device),
425
+ attn_mask.to(device=device),
426
+ )
427
+ input_kwargs = dict(
428
+ n_kv_head=num_kv_heads,
429
+ head_size=head_size,
430
+ mixed_precision=mixed_precision,
431
+ )
432
+
433
+ if return_debug_tensors:
434
+ output_nki, *debug_tensors = flash_attn_varlen_nkifunc(
435
+ *input_args, **input_kwargs)
436
+ else:
437
+ output_nki = flash_attn_varlen_nkifunc(*input_args, **input_kwargs)
438
+ debug_tensors = []
439
+
440
+ output_nki = torch.tensor(output_nki).cpu()
441
+ debug_tensors = [torch.tensor(dt).cpu() for dt in debug_tensors]
442
+
443
+ num_actual_tokens = sum(query_lens)
444
+ print(f"{num_actual_tokens=}")
445
+ # - o: shape (bs, n_heads, seq_q, d) -> (bs, seq_q, n_heads, d)
446
+ output_nki = output_nki.permute(
447
+ 0, 2, 1, 3)[:, :, :, :head_size].cpu()[0, :num_actual_tokens, :, :]
448
+ output_ref_padded = F.pad(
449
+ output_ref,
450
+ (0, 0, 0, 0, 0, 0, 0, max_num_queries_padded - output_ref.shape[0]),
451
+ "constant",
452
+ 0,
453
+ )
454
+ output_ref = output_ref_padded.transpose(0, 1)[0, :num_actual_tokens, :, :]
455
+
456
+ torch.testing.assert_close(output_nki, output_ref, atol=1e-2, rtol=0)
vllm/attention/ops/nki_flash_attn.py
new
+669
@@ -0,0 +1,669 @@
1
+from dataclasses import dataclass
2
+
3
+import neuronxcc.nki.isa as nisa
4
+import neuronxcc.nki.language as nl
5
+import numpy as np
6
+from neuronxcc import nki
7
+from neuronxcc.nki.language import par_dim
8
+
9
+
10
+@dataclass(frozen=True)
11
+class FlashConfig:
12
+ """
13
+ Config class for flash attention with default values
14
+ """
15
+
16
+ seq_tile_size: int = 2048
17
+ should_transpose_v: bool = False
18
+
19
+ __annotations__ = {
20
+ "seq_tile_size": int,
21
+ "should_transpose_v": bool,
22
+ }
23
+
24
+
25
+@nki.jit
26
+def transpose_p_local(p_local_transposed,
27
+ p_local,
28
+ LARGE_TILE_SZ,
29
+ forward_mask,
30
+ B_F_SIZE=512):
31
+ for i in nl.affine_range(LARGE_TILE_SZ // B_F_SIZE):
32
+ if nisa.get_nc_version() == nisa.nc_version.gen3:
33
+ p_local_t_tmp = nl.ndarray((par_dim(128), B_F_SIZE),
34
+ buffer=nl.sbuf,
35
+ dtype=p_local.dtype)
36
+ else:
37
+ p_local_t_tmp = nl.ndarray((par_dim(128), B_F_SIZE),
38
+ buffer=nl.psum,
39
+ dtype=np.float32)
40
+
41
+ for j in nl.affine_range(B_F_SIZE // 128):
42
+ j_128_slice = nl.ds(j * 128, 128)
43
+ i_j_128_slice = nl.ds(i * B_F_SIZE + j * 128, 128)
44
+
45
+ if nisa.get_nc_version() == nisa.nc_version.gen3:
46
+ p_local_t_tmp[:, j_128_slice] = nisa.dma_transpose(
47
+ p_local[:, i_j_128_slice], mask=forward_mask)
48
+ else:
49
+ p_local_t_tmp[:, j_128_slice] = nisa.nc_transpose(
50
+ p_local[:, i_j_128_slice], mask=forward_mask)
51
+
52
+ p_local_transposed[:, nl.ds(i * B_F_SIZE, B_F_SIZE)] = nl.copy(
53
+ p_local_t_tmp, dtype=p_local_transposed.dtype, mask=forward_mask)
54
+
55
+
56
+@nki.jit
57
+def _flash_attention_core(
58
+ q_local_tile,
59
+ k,
60
+ v,
61
+ q_h_per_k_h,
62
+ seqlen_q,
63
+ nheads,
64
+ o_buffer,
65
+ l_buffer,
66
+ m_buffer,
67
+ batch_id,
68
+ head_id,
69
+ gqa_head_idx,
70
+ q_tile_idx,
71
+ local_k_large_tile_idx,
72
+ kernel_dtype,
73
+ acc_type,
74
+ flash_config: FlashConfig,
75
+ use_causal_mask=False,
76
+ continuous_batching_mask=None,
77
+ initialize=False,
78
+ B_P_SIZE=128,
79
+ B_F_SIZE=512,
80
+ B_D_SIZE=128,
81
+ dropout_p=0.0,
82
+ dropout_p_tensor=None,
83
+ seed_tensor=None,
84
+ logit_bias_tile=None,
85
+ qk_res_buffer=None,
86
+):
87
+ """
88
+ The flash attention core function to calculate self attention between a tile
89
+ of q and a block of K and V.
90
+ The q_local_tile has (B_P_SIZE, B_F_SIZE), which is loaded into the SBUF
91
+ already. The block size of K and V
92
+ is defined in the seq_tile_size of the flash_config. The results are stored
93
+ in the following three buffers
94
+ o_buffer: (B_P_SIZE, d)
95
+ l_buffer: (B_P_SIZE, 1)
96
+ m_buffer: (B_P_SIZE, 1)
97
+ """
98
+ LARGE_TILE_SZ = flash_config.seq_tile_size
99
+ num_k_tile_per_large_tile = LARGE_TILE_SZ // B_F_SIZE
100
+ seqlen_k = k.shape[-1]
101
+ seqlen_q // B_P_SIZE
102
+ seqlen_k // B_F_SIZE
103
+
104
+ # TODO : support logit_bias with continuous_batching_mask
105
+ assert not use_causal_mask, "causal mask is not supported."
106
+ assert (continuous_batching_mask
107
+ is not None), "continuous_batching_mask input is required."
108
+ if continuous_batching_mask is not None:
109
+ assert (logit_bias_tile is
110
+ None), "continuous_batching_mask does not support logit_bias!"
111
+
112
+ # mask are used to only apply computation to the lower half of the matrix,
113
+ # which reduce the arthimetic intensity by half
114
+ forward_mask = (q_tile_idx * B_P_SIZE >= local_k_large_tile_idx *
115
+ LARGE_TILE_SZ if use_causal_mask else None)
116
+
117
+ qk_res_buf = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
118
+ buffer=nl.sbuf,
119
+ dtype=acc_type)
120
+ max_local = nl.ndarray((par_dim(B_P_SIZE), num_k_tile_per_large_tile),
121
+ dtype=acc_type)
122
+ for k_i in nl.affine_range(num_k_tile_per_large_tile):
123
+ k_i_b_f_slice = nl.ds(k_i * B_F_SIZE, B_F_SIZE)
124
+
125
+ qk_psum = nl.zeros((par_dim(B_P_SIZE), B_F_SIZE),
126
+ dtype=np.float32,
127
+ buffer=nl.psum) # (128, 512)
128
+ qk_psum[:, :] = nl.matmul(q_local_tile,
129
+ k[:, k_i_b_f_slice],
130
+ transpose_x=True,
131
+ mask=None) # (p(128), 512)
132
+
133
+ qk_res_buf[:, k_i_b_f_slice] = nl.where(
134
+ continuous_batching_mask[:, k_i_b_f_slice],
135
+ qk_psum[:, nl.ds(0, B_F_SIZE)],
136
+ -9984.0,
137
+ dtype=acc_type,
138
+ )
139
+
140
+ # Calculate max of the current tile
141
+ max_local[:, k_i] = nisa.tensor_reduce(
142
+ np.max,
143
+ qk_res_buf[:, k_i_b_f_slice],
144
+ axis=(1, ),
145
+ dtype=acc_type,
146
+ negate=False,
147
+ mask=forward_mask,
148
+ )
149
+
150
+ if qk_res_buffer is not None:
151
+ qk_res_buffer[:, :] = nl.copy(qk_res_buf[:, :])
152
+
153
+ max_ = nisa.tensor_reduce(
154
+ np.max,
155
+ max_local[:, :],
156
+ axis=(1, ),
157
+ dtype=acc_type,
158
+ negate=False,
159
+ mask=forward_mask,
160
+ )
161
+
162
+ o_previous_scaled = nl.ndarray((par_dim(B_P_SIZE), B_D_SIZE),
163
+ dtype=o_buffer.dtype)
164
+
165
+ if initialize:
166
+ m_buffer[:, 0] = nl.copy(max_)
167
+ m_current = max_
168
+ else:
169
+ m_previous = nl.copy(m_buffer[:, 0])
170
+ m_buffer[:, 0] = nl.maximum(m_previous, max_,
171
+ mask=forward_mask) # (128,1)
172
+
173
+ m_current = m_buffer[:, 0]
174
+ # Compute scaling factor
175
+ alpha = nisa.activation(
176
+ np.exp,
177
+ m_previous,
178
+ bias=-1 * m_current,
179
+ scale=1.0,
180
+ mask=forward_mask,
181
+ )
182
+ o_previous_scaled[...] = nl.multiply(o_buffer[:, :],
183
+ alpha,
184
+ mask=forward_mask)
185
+
186
+ p_local = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
187
+ dtype=kernel_dtype)
188
+ REDUCTION_TILE = min(2048, LARGE_TILE_SZ // 2)
189
+
190
+ p_partial_sum = nl.ndarray(
191
+ (par_dim(B_P_SIZE), LARGE_TILE_SZ // REDUCTION_TILE), dtype=acc_type)
192
+
193
+ for k_r_i in nl.affine_range(LARGE_TILE_SZ // REDUCTION_TILE):
194
+ k_r_i_reduce_slice = nl.ds(k_r_i * REDUCTION_TILE, REDUCTION_TILE)
195
+
196
+ # compute exp(qk - max)
197
+ # Compute partial row - tile sum of exp(qk - max))
198
+ # FIXME : Use activation accumulate to accumulate over k_r_i loop ?
199
+ p_local[:, k_r_i_reduce_slice] = nisa.activation_reduce(
200
+ np.exp,
201
+ qk_res_buf[:, k_r_i_reduce_slice],
202
+ bias=-1 * m_current,
203
+ scale=1.0,
204
+ reduce_op=nl.add,
205
+ reduce_res=p_partial_sum[:, k_r_i],
206
+ dtype=kernel_dtype,
207
+ mask=forward_mask,
208
+ )
209
+
210
+ ps = nl.sum(p_partial_sum, axis=1, dtype=acc_type, mask=forward_mask)
211
+
212
+ p_local_transposed = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
213
+ dtype=kernel_dtype)
214
+ transpose_p_local(
215
+ p_local_transposed=p_local_transposed,
216
+ p_local=p_local,
217
+ LARGE_TILE_SZ=LARGE_TILE_SZ,
218
+ forward_mask=forward_mask,
219
+ B_F_SIZE=B_F_SIZE,
220
+ )
221
+
222
+ pv_psum = nl.zeros((par_dim(B_P_SIZE), B_D_SIZE),
223
+ dtype=np.float32,
224
+ buffer=nl.psum)
225
+ for k_i in nl.affine_range(LARGE_TILE_SZ // B_P_SIZE):
226
+ pv_psum[:, :] += nl.matmul(
227
+ p_local_transposed[:, nl.ds(k_i * B_P_SIZE, B_P_SIZE)],
228
+ v[k_i, :, :],
229
+ transpose_x=True,
230
+ mask=forward_mask,
231
+ ) # (128, 128) (p(Br), d)
232
+
233
+ if initialize:
234
+ o_buffer[:, :] = nl.copy(pv_psum[:, :])
235
+ l_buffer[:, 0] = nl.add(nl.log(ps), max_)
236
+ else:
237
+ o_buffer[:, :] = nl.add(o_previous_scaled, pv_psum, mask=forward_mask)
238
+
239
+ l_prev = l_buffer[:, 0]
240
+ l_exp = nl.add(
241
+ nl.exp(
242
+ nl.subtract(l_prev, m_current, mask=forward_mask),
243
+ mask=forward_mask,
244
+ ),
245
+ ps,
246
+ mask=forward_mask,
247
+ )
248
+ l_buffer[:, 0] = nl.add(m_current,
249
+ nl.log(l_exp, mask=forward_mask),
250
+ mask=forward_mask)
251
+
252
+
253
+@nki.jit
254
+def load_v_tile(v_hbm_tile, cur_v_tile, j, v_i, config):
255
+ LARGE_TILE_SZ = config.seq_tile_size
256
+ B_P_SIZE = 128
257
+
258
+ if not config.should_transpose_v:
259
+ cur_v_tile[v_i, :, :] = nl.load(
260
+ v_hbm_tile[nl.ds(j * LARGE_TILE_SZ + B_P_SIZE * v_i, B_P_SIZE), :],
261
+ dtype=cur_v_tile.dtype,
262
+ )
263
+ return
264
+
265
+ if nisa.get_nc_version() == nisa.nc_version.gen3:
266
+ cur_v_tile_transposed = nisa.dma_transpose(
267
+ v_hbm_tile[:,
268
+ nl.ds(j * LARGE_TILE_SZ + B_P_SIZE * v_i, B_P_SIZE)])
269
+ cur_v_tile[v_i, :, :] = nisa.tensor_copy(cur_v_tile_transposed,
270
+ dtype=cur_v_tile.dtype)
271
+ return
272
+
273
+ cur_v_tile[v_i, :, :] = nl.load_transpose2d(
274
+ v_hbm_tile[:, nl.ds(j * LARGE_TILE_SZ + B_P_SIZE * v_i, B_P_SIZE)],
275
+ dtype=cur_v_tile.dtype,
276
+ )
277
+
278
+
279
+@nki.jit
280
+def flash_paged_attention(
281
+ query,
282
+ key,
283
+ value,
284
+ key_cache,
285
+ value_cache,
286
+ block_tables,
287
+ mask,
288
+ softmax_scale=None,
289
+ mixed_precision=True,
290
+ config=None,
291
+ return_debug_tensors=False,
292
+):
293
+ """
294
+ Flash PagedAttention Forward Kernel.
295
+ - PagedAttention Paper: https://arxiv.org/abs/2309.06180
296
+ - Chunked Prefill Paper: https://arxiv.org/abs/2403.02310
297
+
298
+ IO tensor layouts:
299
+ - query: shape (1, n_heads, d, seq_q)
300
+ - key: shape (1, n_kv_heads, d, seq_k)
301
+ - value: shape (1, n_kv_heads, seq_v, d)
302
+ - key_cache: (num_blocks, block_size, n_kv_heads, d)
303
+ - value_cache: (num_blocks, block_size, n_kv_heads, d)
304
+ - block_tables: (num_active_blocks, )
305
+ - mask: (seq_q, num_active_blocks * block_size)
306
+ - o: shape (1, n_heads, seq_q, d)
307
+ - l_m: shape (1, n_heads, seq_q, 2)
308
+
309
+ - This kernel requires seq_k == seq_v
310
+ - We use continuous batching by default, so the batch dimension is
311
+ always 1, and different requests are concatenated along sequence
312
+ dimension.
313
+ - We use paged cache blocks (key_cache, value_cache) to store KV cache.
314
+
315
+ IO tensor dtypes:
316
+ - This kernel assumes all IO tensors have the same dtype except for
317
+ block_tables (int32) and mask (int32)
318
+ - If mixed_percision is True, then all Tensor Engine operation will be
319
+ performed in bfloat16 and accumulation will be performed in float32.
320
+ Otherwise the intermediates will be in the same type as the inputs.
321
+
322
+ Compile-time Constants:
323
+ - softmax_scale: scaling for softmax, is None, default is `1.0/(d**0.5)`
324
+ - mixed_precision: flag to set non-matmul ops in fp32 precision, default
325
+ is set to `true`, if false, we use same precision as input types
326
+ - config: Instance of dataclass :class:`nki.kernels.attention.FlashConfig`
327
+ with Performance config parameters for flash attention with default
328
+ values
329
+ seq_tile_size: `default=2048`, size of the kv tile size for attention
330
+ computation reduction
331
+
332
+ GQA support Notes:
333
+ the spmd kernel for launching kernel should be on kv_heads instead of
334
+ nheads
335
+
336
+ Example usage:
337
+ MHA: q: [b, h, d, s], k: [b, h, d, s], v: [b, h, s, d]
338
+ usage: `flash_fwd[b, h](q, k, v, ...)`
339
+ GQA: q: [b, h, d, s], k: [b, kv_h, d, s], v: [b, kv_h, s, d]
340
+ usage: `flash_fwd[b, kv_h](q, k, v, ...)`
341
+ """
342
+ config = config or FlashConfig()
343
+ B_F_SIZE = 512
344
+ B_P_SIZE = 128
345
+ b, h, d, seqlen_q = query.shape
346
+ B_D_SIZE = d
347
+ LARGE_TILE_SZ = config.seq_tile_size
348
+ n_tile_q = seqlen_q // B_P_SIZE # since q will be loaded on tensor engine
349
+ num_blocks, block_size, k_h, _ = key_cache.shape
350
+ q_h_per_k_h = h // k_h
351
+ assert tuple(key_cache.shape) == (
352
+ num_blocks,
353
+ block_size,
354
+ k_h,
355
+ d,
356
+ ), "Input shape mismatch!"
357
+ assert tuple(value_cache.shape) == (
358
+ num_blocks,
359
+ block_size,
360
+ k_h,
361
+ d,
362
+ ), "Input shape mismatch!"
363
+ assert b == 1, f"invalid batch size {b=}"
364
+ assert d <= 128, f" we do not support head_dim > 128, got head dim {d}"
365
+ kernel_dtype = nl.bfloat16 if mixed_precision else query.dtype
366
+ acc_type = np.dtype(np.float32) if mixed_precision else kernel_dtype
367
+
368
+ o = nl.ndarray((b, h, seqlen_q, d),
369
+ dtype=query.dtype,
370
+ buffer=nl.shared_hbm)
371
+ hbm_l_buffer, hbm_m_buffer, hbm_qk_res, qk_res_buffer = (
372
+ None,
373
+ None,
374
+ None,
375
+ None,
376
+ )
377
+ if return_debug_tensors:
378
+ hbm_l_buffer = nl.ndarray((b, h, seqlen_q),
379
+ dtype=acc_type,
380
+ buffer=nl.shared_hbm)
381
+ hbm_m_buffer = nl.ndarray((b, h, seqlen_q),
382
+ dtype=acc_type,
383
+ buffer=nl.shared_hbm)
384
+ hbm_qk_res = nl.ndarray((b, h, B_P_SIZE, seqlen_q),
385
+ dtype=acc_type,
386
+ buffer=nl.shared_hbm)
387
+ qk_res_buffer = nl.zeros(
388
+ (n_tile_q, q_h_per_k_h, par_dim(B_P_SIZE), seqlen_q),
389
+ dtype=acc_type,
390
+ buffer=nl.sbuf,
391
+ lazy_initialization=True,
392
+ )
393
+
394
+ assert (
395
+ nl.program_ndim() == 2
396
+ ), f"Expect spmd grid with 2 dimensions, got {nl.program_ndim()} instead!"
397
+ batch_id = nl.program_id(axis=0)
398
+ head_id = nl.program_id(axis=1)
399
+
400
+ softmax_scale = softmax_scale or (1.0 / (d**0.5))
401
+
402
+ (num_active_blocks, ) = block_tables.shape
403
+ context_kv_len = num_active_blocks * block_size
404
+ assert (config.seq_tile_size >= 512
405
+ ), f" seq tile_size {config.seq_tile_size} cannot be less than 512"
406
+ assert (context_kv_len % LARGE_TILE_SZ == 0
407
+ ), f"Need {context_kv_len=} to be divisible by {LARGE_TILE_SZ=}"
408
+ assert (
409
+ LARGE_TILE_SZ % B_P_SIZE == 0
410
+ ), f"Need LARGE_TILE_SZ ({LARGE_TILE_SZ}) to be divisible by {B_P_SIZE=}"
411
+ assert (B_P_SIZE % block_size == 0
412
+ ), f"Need B_P_SIZE ({B_P_SIZE}) to be divisible by {block_size=}"
413
+ num_large_k_tile = context_kv_len // LARGE_TILE_SZ
414
+ num_blocks_per_large_tile = LARGE_TILE_SZ // block_size
415
+ assert (num_blocks_per_large_tile <= B_P_SIZE
416
+ ), f"The number of blocks in each large tile " \
417
+ f"({num_blocks_per_large_tile}) shouldn't exceed partition size {B_P_SIZE}"
418
+
419
+ block_tables_sbuf = nl.full((par_dim(B_P_SIZE), num_large_k_tile),
420
+ 0,
421
+ dtype=np.int32,
422
+ buffer=nl.sbuf)
423
+ for j in nl.affine_range(num_large_k_tile):
424
+ i_p = nl.arange(num_blocks_per_large_tile)[:, None]
425
+ block_tables_sbuf[i_p, j] = nl.load(
426
+ block_tables[j * num_blocks_per_large_tile + i_p], dtype=np.int32)
427
+
428
+ # Global Flash Attention accumulators
429
+ o_buffer = nl.zeros(
430
+ (n_tile_q, q_h_per_k_h, par_dim(B_P_SIZE), d),
431
+ dtype=acc_type,
432
+ buffer=nl.sbuf,
433
+ lazy_initialization=True,
434
+ )
435
+ l_buffer = nl.zeros(
436
+ (par_dim(B_P_SIZE), n_tile_q, q_h_per_k_h),
437
+ dtype=acc_type,
438
+ buffer=nl.sbuf,
439
+ lazy_initialization=True,
440
+ )
441
+ m_buffer = nl.zeros(
442
+ (n_tile_q, q_h_per_k_h, par_dim(B_P_SIZE), 1),
443
+ dtype=acc_type,
444
+ buffer=nl.sbuf,
445
+ lazy_initialization=True,
446
+ )
447
+
448
+ for j in nl.sequential_range(0, num_large_k_tile):
449
+ cur_k_tile = nl.ndarray((par_dim(B_D_SIZE), LARGE_TILE_SZ),
450
+ dtype=kernel_dtype)
451
+ cur_v_tile = nl.ndarray(
452
+ (LARGE_TILE_SZ // B_P_SIZE, par_dim(B_P_SIZE), B_D_SIZE),
453
+ dtype=kernel_dtype,
454
+ )
455
+
456
+ for k_i in nl.affine_range(num_blocks_per_large_tile):
457
+ loaded = nl.load(key_cache[block_tables_sbuf[k_i, j], :,
458
+ head_id, :])
459
+ cur_k_tile[:, nl.ds(k_i *
460
+ block_size, block_size)] = nl.transpose(loaded)
461
+
462
+ load_tile_size = B_P_SIZE
463
+ num_blocks_per_partition = load_tile_size // block_size
464
+ for partition_idx in nl.affine_range(LARGE_TILE_SZ // load_tile_size):
465
+ for block_in_partition in nl.affine_range(
466
+ num_blocks_per_partition):
467
+ v_i = (partition_idx * num_blocks_per_partition +
468
+ block_in_partition)
469
+ loaded_v = nl.load(value_cache[block_tables_sbuf[v_i, j], :,
470
+ head_id, :])
471
+ cur_v_tile[partition_idx,
472
+ nl.ds(block_in_partition *
473
+ block_size, block_size), :, ] = loaded_v
474
+
475
+ cur_mask = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
476
+ dtype=mask.dtype)
477
+ for m_i in nl.affine_range(LARGE_TILE_SZ // B_F_SIZE):
478
+ cur_mask[:, nl.ds(m_i * B_F_SIZE, B_F_SIZE)] = nl.load(
479
+ mask[:, nl.ds(j * LARGE_TILE_SZ + m_i * B_F_SIZE, B_F_SIZE)])
480
+
481
+ for i_q_h in nl.affine_range(q_h_per_k_h):
482
+ for i in nl.affine_range(n_tile_q):
483
+ q_tile = nl.ndarray((B_D_SIZE, B_P_SIZE), dtype=kernel_dtype)
484
+ q_hbm_tile = query[batch_id, head_id * q_h_per_k_h + i_q_h]
485
+ q_sbuf_tile = nl.load(
486
+ q_hbm_tile[:, nl.ds(i * B_P_SIZE, B_P_SIZE)],
487
+ dtype=kernel_dtype,
488
+ ) # load (d, 128) tile in SBUF
489
+ q_tile[:, :] = q_sbuf_tile * softmax_scale
490
+
491
+ _flash_attention_core(
492
+ q_local_tile=q_tile,
493
+ k=cur_k_tile,
494
+ v=cur_v_tile,
495
+ q_h_per_k_h=q_h_per_k_h,
496
+ seqlen_q=seqlen_q,
497
+ nheads=h,
498
+ o_buffer=o_buffer[i, i_q_h],
499
+ l_buffer=l_buffer[:, i, i_q_h],
500
+ m_buffer=m_buffer[i, i_q_h],
501
+ batch_id=batch_id,
502
+ head_id=head_id,
503
+ gqa_head_idx=i_q_h,
504
+ q_tile_idx=i,
505
+ local_k_large_tile_idx=j,
506
+ kernel_dtype=kernel_dtype,
507
+ acc_type=acc_type,
508
+ flash_config=config,
509
+ use_causal_mask=False,
510
+ continuous_batching_mask=cur_mask,
511
+ initialize=j == 0,
512
+ B_P_SIZE=B_P_SIZE,
513
+ B_F_SIZE=B_F_SIZE,
514
+ B_D_SIZE=B_D_SIZE,
515
+ dropout_p=0.0,
516
+ dropout_p_tensor=None,
517
+ seed_tensor=None,
518
+ logit_bias_tile=None,
519
+ )
520
+
521
+ # compute attention between input query, key and value
522
+ if key is not None and value is not None:
523
+ B_F_SIZE = seqlen_q
524
+ LARGE_TILE_SZ = seqlen_q
525
+ active_config = FlashConfig(
526
+ seq_tile_size=LARGE_TILE_SZ,
527
+ should_transpose_v=config.should_transpose_v,
528
+ )
529
+
530
+ cur_k_tile = nl.ndarray((par_dim(B_D_SIZE), LARGE_TILE_SZ),
531
+ dtype=kernel_dtype)
532
+ cur_v_tile = nl.ndarray(
533
+ (LARGE_TILE_SZ // B_P_SIZE, par_dim(B_P_SIZE), B_D_SIZE),
534
+ dtype=kernel_dtype,
535
+ )
536
+
537
+ cur_k_tile[:, :] = nl.load(key[batch_id, head_id, :, :])
538
+
539
+ load_tile_size = B_P_SIZE
540
+ v_hbm_tile = value[batch_id, head_id]
541
+ for v_i in nl.affine_range(LARGE_TILE_SZ // load_tile_size):
542
+ load_v_tile(
543
+ v_hbm_tile=v_hbm_tile,
544
+ cur_v_tile=cur_v_tile,
545
+ j=0,
546
+ v_i=v_i,
547
+ config=active_config,
548
+ )
549
+
550
+ cur_mask = nl.ndarray((par_dim(B_P_SIZE), B_F_SIZE), dtype=mask.dtype)
551
+ cur_mask[:, :] = nl.load(mask[:, nl.ds(context_kv_len, B_F_SIZE)])
552
+
553
+ for i_q_h in nl.affine_range(q_h_per_k_h):
554
+ for i in nl.affine_range(n_tile_q):
555
+ q_tile = nl.ndarray((B_D_SIZE, B_P_SIZE), dtype=kernel_dtype)
556
+ q_hbm_tile = query[batch_id, head_id * q_h_per_k_h + i_q_h]
557
+ q_sbuf_tile = nl.load(
558
+ q_hbm_tile[:, nl.ds(i * B_P_SIZE, B_P_SIZE)],
559
+ dtype=kernel_dtype,
560
+ ) # load (d, 128) tile in SBUF
561
+ q_tile[:, :] = q_sbuf_tile * softmax_scale
562
+ _flash_attention_core(
563
+ q_local_tile=q_tile,
564
+ k=cur_k_tile,
565
+ v=cur_v_tile,
566
+ q_h_per_k_h=q_h_per_k_h,
567
+ seqlen_q=seqlen_q,
568
+ nheads=h,
569
+ o_buffer=o_buffer[i, i_q_h],
570
+ l_buffer=l_buffer[:, i, i_q_h],
571
+ m_buffer=m_buffer[i, i_q_h],
572
+ batch_id=batch_id,
573
+ head_id=head_id,
574
+ gqa_head_idx=i_q_h,
575
+ q_tile_idx=i,
576
+ local_k_large_tile_idx=0,
577
+ kernel_dtype=kernel_dtype,
578
+ acc_type=acc_type,
579
+ flash_config=active_config,
580
+ use_causal_mask=False,
581
+ continuous_batching_mask=cur_mask,
582
+ initialize=False,
583
+ B_P_SIZE=B_P_SIZE,
584
+ B_F_SIZE=B_F_SIZE,
585
+ B_D_SIZE=B_D_SIZE,
586
+ dropout_p=0.0,
587
+ dropout_p_tensor=None,
588
+ seed_tensor=None,
589
+ logit_bias_tile=None,
590
+ qk_res_buffer=qk_res_buffer[i, i_q_h]
591
+ if qk_res_buffer is not None else None,
592
+ )
593
+
594
+ # -- -- -- -- write output to buffer on HBM -- -- -- -- -- -- #
595
+ for i_q_h in nl.affine_range(q_h_per_k_h):
596
+ for i in nl.affine_range(n_tile_q):
597
+ out = nl.multiply(
598
+ o_buffer[i, i_q_h, :, :],
599
+ nl.exp(m_buffer[i, i_q_h, :, :] - l_buffer[:, i, i_q_h]),
600
+ dtype=kernel_dtype,
601
+ )
602
+
603
+ nl.store(
604
+ o[batch_id, head_id * q_h_per_k_h + i_q_h,
605
+ nl.ds(i * B_P_SIZE, B_P_SIZE), :, ],
606
+ out,
607
+ )
608
+ # maximum and summation statistics
609
+ if return_debug_tensors:
610
+ nl.store(
611
+ hbm_m_buffer[batch_id, head_id * q_h_per_k_h + i_q_h,
612
+ nl.ds(i * B_P_SIZE, B_P_SIZE), ],
613
+ m_buffer[i, i_q_h, :, :],
614
+ )
615
+ nl.store(
616
+ hbm_l_buffer[batch_id, head_id * q_h_per_k_h + i_q_h,
617
+ nl.ds(i * B_P_SIZE, B_P_SIZE), ],
618
+ l_buffer[:, i, i_q_h],
619
+ )
620
+ nl.store(
621
+ hbm_qk_res[batch_id, head_id * q_h_per_k_h + i_q_h, :, :],
622
+ qk_res_buffer[batch_id, i_q_h, :, :],
623
+ )
624
+
625
+ if return_debug_tensors:
626
+ return o, hbm_m_buffer, hbm_l_buffer, hbm_qk_res
627
+ return o
628
+
629
+
630
+def flash_attn_varlen_nkifunc(
631
+ query,
632
+ key,
633
+ value,
634
+ key_cache,
635
+ value_cache,
636
+ block_table,
637
+ attn_mask,
638
+ n_kv_head=None,
639
+ head_size=None,
640
+ B_P_SIZE=128,
641
+ LARGE_TILE_SZ=2048,
642
+ return_debug_tensors=False,
643
+ mixed_precision=True,
644
+):
645
+ config = FlashConfig(
646
+ seq_tile_size=LARGE_TILE_SZ,
647
+ should_transpose_v=False,
648
+ )
649
+ kwargs = dict(
650
+ query=query,
651
+ key=key,
652
+ value=value,
653
+ key_cache=key_cache,
654
+ value_cache=value_cache,
655
+ block_tables=block_table,
656
+ mask=attn_mask,
657
+ softmax_scale=1.0 / (head_size**0.5),
658
+ config=config,
659
+ mixed_precision=mixed_precision,
660
+ return_debug_tensors=return_debug_tensors,
661
+ )
662
+ _, n_kv_head, _, _ = key.shape
663
+
664
+ if return_debug_tensors:
665
+ o, *debug_tensors = flash_paged_attention[1, n_kv_head](**kwargs)
666
+ return o, *debug_tensors
667
+ else:
668
+ o = flash_paged_attention[1, n_kv_head](**kwargs)
669
+ return o