@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e92694b6fe
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Neuron][Kernel] Support Longer Sequences in NKI-based Flash PagedAttention and Improve Efficiency (#12921)

Signed-off-by: Lingfan Yu <lingfany@amazon.com>

Lingfan Yu committed Feb 11, 2025 at 21:12 UTC e92694b6fe264a85371317295bca6643508034ef
2 files changed +154 -180
tests/neuron/test_prefix_prefill.py
+67 -51
@@ -1,6 +1,5 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 -import random
3 from typing import Optional
4
5 import pytest
@@ -171,12 +170,22 @@ def ref_context_attention(
170 return output
171
172
173 +@pytest.mark.parametrize(
174 + "block_size, large_tile_size",
175 + [
176 + (32, 2048), # 64 blocks
177 + (32, 4096), # 128 blocks
178 + (32, 8192), # 256 blocks
179 + (64, 8192), # 128 blocks
180 + ],
181 +)
182 @pytest.mark.parametrize(
183 "num_heads,num_queries_per_kv,head_size,mixed_precision",
184 [
185 (4, 2, 8, False),
186 (4, 2, 8, True),
187 (32, 8, 64, True),
188 + (16, 2, 128, True),
189 ],
190 )
191 @torch.inference_mode()
@@ -184,6 +193,8 @@ def test_contexted_kv_attention(
193 num_heads: int,
194 num_queries_per_kv: int,
195 head_size: int,
196 + block_size: int,
197 + large_tile_size,
198 mixed_precision: bool,
199 ) -> None:
200 import os
@@ -192,40 +203,46 @@ def test_contexted_kv_attention(
203
204 from vllm.attention.ops.nki_flash_attn import flash_attn_varlen_nkifunc
205
206 + assert large_tile_size % block_size == 0
207 +
208 device = xm.xla_device()
209
197 - os.environ["NEURON_CC_FLAGS"] = (
198 - " --model-type=transformer -O1 "
199 - " --internal-hlo2tensorizer-options='--verify-hlo' ")
210 + compiler_flags = [
211 + "--model-type=transformer -O1",
212 + "--internal-hlo2tensorizer-options='--verify-hlo'",
213 + "--retry_failed_compilation",
214 + ]
215 + compiler_flags_str = " ".join(compiler_flags)
216 + os.environ["NEURON_CC_FLAGS"] = compiler_flags_str
217
201 - random.seed(0)
218 torch.manual_seed(0)
219 torch.set_printoptions(sci_mode=False)
220
205 - min_ctx_len = 2
206 - max_ctx_len = 64
207 - min_query_len = 2
208 - max_query_len = 64
209 - prefill_batch_size = 2
210 - decode_batch_size = 6
221 + min_ctx_len = 32
222 + max_ctx_len = 1024
223 + min_query_len = 16
224 + max_query_len = 512
225 + prefill_batch_size = 4
226 + decode_batch_size = 12
227 batch_size = prefill_batch_size + decode_batch_size
212 - block_size = 32
228 max_model_len = (max_query_len + max_ctx_len) * 4
229
230 max_block_per_request = max_model_len // block_size
231 dtype = torch.float32
232 cache_size = (batch_size * max_block_per_request) + 2
218 - ctx_lens = [
219 - random.randint(min_ctx_len, max_ctx_len)
220 - for _ in range(prefill_batch_size)
221 - ] + [
222 - random.randint(min_ctx_len, max_ctx_len)
223 - for _ in range(decode_batch_size)
224 - ]
225 - query_lens = [
226 - random.randint(min_query_len, max_query_len)
227 - for _ in range(prefill_batch_size)
228 - ] + [1 for _ in range(decode_batch_size)]
233 + prefill_ctx_lens = torch.randint(min_ctx_len,
234 + max_ctx_len + 1, (prefill_batch_size, ),
235 + dtype=torch.long).tolist()
236 + decode_ctx_lens = torch.randint(min_ctx_len,
237 + max_ctx_len + 1, (decode_batch_size, ),
238 + dtype=torch.long).tolist()
239 + ctx_lens = prefill_ctx_lens + decode_ctx_lens
240 + query_lens = torch.randint(
241 + min_query_len,
242 + max_query_len + 1,
243 + (prefill_batch_size, ),
244 + dtype=torch.long,
245 + ).tolist() + [1 for _ in range(decode_batch_size)]
246 seq_lens = [a + b for a, b in zip(query_lens, ctx_lens)]
247 num_kv_heads = num_heads // num_queries_per_kv
248
@@ -254,7 +271,6 @@ def test_contexted_kv_attention(
271 values = values[torch.randperm(cache_size)]
272 block_table = values[:batch_size * max_block_per_request].view(
273 batch_size, max_block_per_request)
257 - torch.tensor(seq_lens, dtype=torch.long)
274 b_ctx_len = torch.tensor(ctx_lens, dtype=torch.long)
275 b_start_loc = torch.cumsum(torch.tensor([0] + query_lens[:-1],
276 dtype=torch.long),
@@ -311,9 +327,7 @@ def test_contexted_kv_attention(
327 # build neuron program
328 return_debug_tensors = False
329 B_P_SIZE = 128
314 - LARGE_TILE_SZ = 2048
315 - max_num_queries = (
316 - (sum(query_lens) + block_size - 1) // block_size) * block_size
330 + LARGE_TILE_SZ = large_tile_size
331
332 def get_active_block_tables(block_tables, query_lens, seq_lens, block_size,
333 num_blocks):
@@ -332,26 +346,28 @@ def test_contexted_kv_attention(
346 0,
347 )
348
335 - def shift_bit_length(x):
336 - return 1 << (x - 1).bit_length()
349 + def ceil_div(a, b):
350 + return (a + b - 1) // b
351 +
352 + def pad_to_multiple(a, b):
353 + return ceil_div(a, b) * b
354 +
355 + def pad_to_next_power_of_2(a):
356 + assert a > 0
357 + return 2**int(a - 1).bit_length()
358
359 # calculate input shapes
339 - max_num_queries_shifted = shift_bit_length(max_num_queries)
340 - max_num_queries_factor = B_P_SIZE // max_num_queries_shifted
341 - max_num_queries_padded = max_num_queries_shifted * max_num_queries_factor
342 - assert (max_num_queries_padded == B_P_SIZE
343 - ), "invalid {max_num_queries_padded=}"
360 + max_num_queries = pad_to_multiple(sum(query_lens), block_size)
361 + max_num_queries = pad_to_next_power_of_2(max_num_queries)
362 head_size_padded = B_P_SIZE
363 + assert head_size_padded >= head_size
364 context_lens = torch.tensor(seq_lens) - torch.tensor(query_lens)
346 - num_active_blocks_shifted = shift_bit_length(
347 - ((context_lens + block_size - 1) // block_size).sum().item())
348 - num_active_blocks_factor = (LARGE_TILE_SZ // block_size //
349 - num_active_blocks_shifted)
350 - num_active_blocks = num_active_blocks_shifted * num_active_blocks_factor
351 - assert (num_active_blocks *
352 - block_size) == LARGE_TILE_SZ, "invalid {num_active_blocks=}"
365 + num_active_blocks = ceil_div(context_lens, block_size).sum().item()
366 + num_active_blocks = pad_to_multiple(num_active_blocks,
367 + LARGE_TILE_SZ // block_size)
368 context_kv_len = num_active_blocks * block_size
354 - assert context_kv_len == LARGE_TILE_SZ, f"invalid {context_kv_len=}"
369 + assert (context_kv_len %
370 + LARGE_TILE_SZ == 0), f"invalid context_kv_len={context_kv_len}"
371
372 # pad QKV tensors
373 pad_dims = (
@@ -360,7 +376,7 @@ def test_contexted_kv_attention(
376 0,
377 0,
378 0,
363 - max_num_queries_padded - query.shape[0],
379 + max_num_queries - query.shape[0],
380 )
381 query = F.pad(query, pad_dims, "constant", 0)
382 k = F.pad(k, pad_dims, "constant", 0)
@@ -397,7 +413,7 @@ def test_contexted_kv_attention(
413 0,
414 context_kv_len - prior_mask.shape[1],
415 0,
400 - B_P_SIZE - prior_mask.shape[0],
416 + max_num_queries - prior_mask.shape[0],
417 ),
418 "constant",
419 0,
@@ -406,9 +422,9 @@ def test_contexted_kv_attention(
422 active_mask,
423 (
424 0,
409 - B_P_SIZE - active_mask.shape[1],
425 + max_num_queries - active_mask.shape[1],
426 0,
411 - B_P_SIZE - active_mask.shape[0],
427 + max_num_queries - active_mask.shape[0],
428 ),
429 "constant",
430 0,
@@ -430,6 +446,8 @@ def test_contexted_kv_attention(
446 n_kv_head=num_kv_heads,
447 head_size=head_size,
448 mixed_precision=mixed_precision,
449 + LARGE_TILE_SZ=LARGE_TILE_SZ,
450 + return_debug_tensors=return_debug_tensors,
451 )
452
453 if return_debug_tensors:
@@ -439,17 +457,15 @@ def test_contexted_kv_attention(
457 output_nki = flash_attn_varlen_nkifunc(*input_args, **input_kwargs)
458 debug_tensors = []
459
442 - output_nki = torch.tensor(output_nki).cpu()
460 debug_tensors = [torch.tensor(dt).cpu() for dt in debug_tensors]
461
462 num_actual_tokens = sum(query_lens)
446 - print(f"{num_actual_tokens=}")
463 # - o: shape (bs, n_heads, seq_q, d) -> (bs, seq_q, n_heads, d)
448 - output_nki = output_nki.permute(
449 - 0, 2, 1, 3)[:, :, :, :head_size].cpu()[0, :num_actual_tokens, :, :]
464 + output_nki = output_nki.cpu().permute(0, 2, 1, 3)[:, :, :, :head_size]
465 + output_nki = output_nki[0, :num_actual_tokens, :, :]
466 output_ref_padded = F.pad(
467 output_ref,
452 - (0, 0, 0, 0, 0, 0, 0, max_num_queries_padded - output_ref.shape[0]),
468 + (0, 0, 0, 0, 0, 0, 0, max_num_queries - output_ref.shape[0]),
469 "constant",
470 0,
471 )
vllm/attention/ops/nki_flash_attn.py
+87 -129
@@ -28,7 +28,6 @@ class FlashConfig:
28 def transpose_p_local(p_local_transposed,
29 p_local,
30 LARGE_TILE_SZ,
31 - forward_mask,
31 B_F_SIZE=512):
32 for i in nl.affine_range(LARGE_TILE_SZ // B_F_SIZE):
33 if nisa.get_nc_version() == nisa.nc_version.gen3:
@@ -46,13 +45,13 @@ def transpose_p_local(p_local_transposed,
45
46 if nisa.get_nc_version() == nisa.nc_version.gen3:
47 p_local_t_tmp[:, j_128_slice] = nisa.dma_transpose(
49 - p_local[:, i_j_128_slice], mask=forward_mask)
48 + p_local[:, i_j_128_slice])
49 else:
50 p_local_t_tmp[:, j_128_slice] = nisa.nc_transpose(
52 - p_local[:, i_j_128_slice], mask=forward_mask)
51 + p_local[:, i_j_128_slice])
52
53 p_local_transposed[:, nl.ds(i * B_F_SIZE, B_F_SIZE)] = nl.copy(
55 - p_local_t_tmp, dtype=p_local_transposed.dtype, mask=forward_mask)
54 + p_local_t_tmp, dtype=p_local_transposed.dtype)
55
56
57 @nki.jit
@@ -60,36 +59,25 @@ def _flash_attention_core(
59 q_local_tile,
60 k,
61 v,
63 - q_h_per_k_h,
64 - seqlen_q,
65 - nheads,
62 o_buffer,
63 l_buffer,
64 m_buffer,
69 - batch_id,
70 - head_id,
71 - gqa_head_idx,
65 q_tile_idx,
73 - local_k_large_tile_idx,
66 kernel_dtype,
67 acc_type,
68 flash_config: FlashConfig,
77 - use_causal_mask=False,
78 - continuous_batching_mask=None,
69 + use_causal_mask,
70 + tile_mask,
71 initialize=False,
72 B_P_SIZE=128,
73 B_F_SIZE=512,
74 B_D_SIZE=128,
83 - dropout_p=0.0,
84 - dropout_p_tensor=None,
85 - seed_tensor=None,
86 - logit_bias_tile=None,
75 qk_res_buffer=None,
76 ):
77 """
78 The flash attention core function to calculate self attention between a tile
79 of q and a block of K and V.
92 - The q_local_tile has (B_P_SIZE, B_F_SIZE), which is loaded into the SBUF
80 + The q_local_tile has (B_P_SIZE, B_F_SIZE), which is loaded into the SBUF
81 already. The block size of K and V
82 is defined in the seq_tile_size of the flash_config. The results are stored
83 in the following three buffers
@@ -99,24 +87,9 @@ def _flash_attention_core(
87 """
88 LARGE_TILE_SZ = flash_config.seq_tile_size
89 num_k_tile_per_large_tile = LARGE_TILE_SZ // B_F_SIZE
102 - seqlen_k = k.shape[-1]
103 - seqlen_q // B_P_SIZE
104 - seqlen_k // B_F_SIZE
105 -
106 - # TODO : support logit_bias with continuous_batching_mask
107 - assert not use_causal_mask, "causal mask is not supported."
108 - assert (continuous_batching_mask
109 - is not None), "continuous_batching_mask input is required."
110 - if continuous_batching_mask is not None:
111 - assert (
112 - logit_bias_tile
113 - is None), "continuous_batching_mask does not support logit_bias!"
90
91 # mask are used to only apply computation to the lower half of the matrix,
92 # which reduce the arithmetic intensity by half
117 - forward_mask = (q_tile_idx * B_P_SIZE >= local_k_large_tile_idx *
118 - LARGE_TILE_SZ if use_causal_mask else None)
119 -
93 qk_res_buf = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
94 buffer=nl.sbuf,
95 dtype=acc_type)
@@ -125,20 +98,27 @@ def _flash_attention_core(
98 for k_i in nl.affine_range(num_k_tile_per_large_tile):
99 k_i_b_f_slice = nl.ds(k_i * B_F_SIZE, B_F_SIZE)
100
128 - qk_psum = nl.zeros((par_dim(B_P_SIZE), B_F_SIZE),
129 - dtype=np.float32,
130 - buffer=nl.psum) # (128, 512)
131 - qk_psum[:, :] = nl.matmul(q_local_tile,
132 - k[:, k_i_b_f_slice],
133 - transpose_x=True,
134 - mask=None) # (p(128), 512)
135 -
136 - qk_res_buf[:, k_i_b_f_slice] = nl.where(
137 - continuous_batching_mask[:, k_i_b_f_slice],
138 - qk_psum[:, nl.ds(0, B_F_SIZE)],
139 - -9984.0,
140 - dtype=acc_type,
141 - )
101 + if use_causal_mask:
102 + multiplication_required_selection = (q_tile_idx * B_P_SIZE
103 + >= k_i * B_F_SIZE)
104 + else:
105 + multiplication_required_selection = True
106 +
107 + if multiplication_required_selection:
108 + qk_psum = nl.ndarray((par_dim(B_P_SIZE), B_F_SIZE),
109 + dtype=np.float32,
110 + buffer=nl.psum) # (128, 512)
111 + qk_psum[:, :] = nl.matmul(q_local_tile,
112 + k[:, k_i_b_f_slice],
113 + transpose_x=True) # (p(128), 512)
114 + qk_res_buf[:, k_i_b_f_slice] = nl.where(
115 + tile_mask[:, k_i_b_f_slice],
116 + qk_psum[:, nl.ds(0, B_F_SIZE)],
117 + -9984.0,
118 + dtype=acc_type,
119 + )
120 + else:
121 + qk_res_buf[:, k_i_b_f_slice] = -9984.0
122
123 # Calculate max of the current tile
124 max_local[:, k_i] = nisa.tensor_reduce(
@@ -147,7 +127,6 @@ def _flash_attention_core(
127 axis=(1, ),
128 dtype=acc_type,
129 negate=False,
150 - mask=forward_mask,
130 )
131
132 if qk_res_buffer is not None:
@@ -159,7 +138,6 @@ def _flash_attention_core(
138 axis=(1, ),
139 dtype=acc_type,
140 negate=False,
162 - mask=forward_mask,
141 )
142
143 o_previous_scaled = nl.ndarray((par_dim(B_P_SIZE), B_D_SIZE),
@@ -170,8 +148,7 @@ def _flash_attention_core(
148 m_current = max_
149 else:
150 m_previous = nl.copy(m_buffer[:, 0])
173 - m_buffer[:, 0] = nl.maximum(m_previous, max_,
174 - mask=forward_mask) # (128,1)
151 + m_buffer[:, 0] = nl.maximum(m_previous, max_) # (128,1)
152
153 m_current = m_buffer[:, 0]
154 # Compute scaling factor
@@ -180,11 +157,8 @@ def _flash_attention_core(
157 m_previous,
158 bias=-1 * m_current,
159 scale=1.0,
183 - mask=forward_mask,
160 )
185 - o_previous_scaled[...] = nl.multiply(o_buffer[:, :],
186 - alpha,
187 - mask=forward_mask)
161 + o_previous_scaled[...] = nl.multiply(o_buffer[:, :], alpha)
162
163 p_local = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
164 dtype=kernel_dtype)
@@ -207,10 +181,9 @@ def _flash_attention_core(
181 reduce_op=nl.add,
182 reduce_res=p_partial_sum[:, k_r_i],
183 dtype=kernel_dtype,
210 - mask=forward_mask,
184 )
185
213 - ps = nl.sum(p_partial_sum, axis=1, dtype=acc_type, mask=forward_mask)
186 + ps = nl.sum(p_partial_sum, axis=1, dtype=acc_type)
187
188 p_local_transposed = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
189 dtype=kernel_dtype)
@@ -218,7 +191,6 @@ def _flash_attention_core(
191 p_local_transposed=p_local_transposed,
192 p_local=p_local,
193 LARGE_TILE_SZ=LARGE_TILE_SZ,
221 - forward_mask=forward_mask,
194 B_F_SIZE=B_F_SIZE,
195 )
196
@@ -230,27 +202,20 @@ def _flash_attention_core(
202 p_local_transposed[:, nl.ds(k_i * B_P_SIZE, B_P_SIZE)],
203 v[k_i, :, :],
204 transpose_x=True,
233 - mask=forward_mask,
205 ) # (128, 128) (p(Br), d)
206
207 if initialize:
208 o_buffer[:, :] = nl.copy(pv_psum[:, :])
209 l_buffer[:, 0] = nl.add(nl.log(ps), max_)
210 else:
240 - o_buffer[:, :] = nl.add(o_previous_scaled, pv_psum, mask=forward_mask)
211 + o_buffer[:, :] = nl.add(o_previous_scaled, pv_psum)
212
213 l_prev = l_buffer[:, 0]
214 l_exp = nl.add(
244 - nl.exp(
245 - nl.subtract(l_prev, m_current, mask=forward_mask),
246 - mask=forward_mask,
247 - ),
215 + nl.exp(nl.subtract(l_prev, m_current)),
216 ps,
249 - mask=forward_mask,
217 )
251 - l_buffer[:, 0] = nl.add(m_current,
252 - nl.log(l_exp, mask=forward_mask),
253 - mask=forward_mask)
218 + l_buffer[:, 0] = nl.add(m_current, nl.log(l_exp))
219
220
221 @nki.jit
@@ -279,6 +244,21 @@ def load_v_tile(v_hbm_tile, cur_v_tile, j, v_i, config):
244 )
245
246
247 +@nki.jit
248 +def load_block_tables(block_tables_hbm, num_tiles):
249 + (num_blocks, ) = block_tables_hbm.shape
250 + assert num_blocks % num_tiles == 0
251 + num_blocks_per_tile = num_blocks // num_tiles
252 + block_tables_hbm = block_tables_hbm.reshape(
253 + (num_tiles, num_blocks_per_tile))
254 + block_tables_buffer = nl.load(block_tables_hbm, dtype=nl.int32)
255 + return block_tables_buffer
256 +
257 +
258 +def is_power_of_2(x):
259 + return x > 0 and (x & (x - 1)) == 0
260 +
261 +
262 @nki.jit
263 def flash_paged_attention(
264 query,
@@ -316,24 +296,24 @@ def flash_paged_attention(
296 - We use paged cache blocks (key_cache, value_cache) to store KV cache.
297
298 IO tensor dtypes:
319 - - This kernel assumes all IO tensors have the same dtype except for
299 + - This kernel assumes all IO tensors have the same dtype except for
300 block_tables (int32) and mask (int32)
321 - - If mixed_percision is True, then all Tensor Engine operation will be
322 - performed in bfloat16 and accumulation will be performed in float32.
301 + - If mixed_percision is True, then all Tensor Engine operation will be
302 + performed in bfloat16 and accumulation will be performed in float32.
303 Otherwise the intermediates will be in the same type as the inputs.
304
305 Compile-time Constants:
306 - softmax_scale: scaling for softmax, is None, default is `1.0/(d**0.5)`
307 - mixed_precision: flag to set non-matmul ops in fp32 precision, default
328 - is set to `true`, if false, we use same precision as input types
308 + is set to `true`, if false, we use same precision as input types
309 - config: Instance of dataclass :class:`nki.kernels.attention.FlashConfig`
310 with Performance config parameters for flash attention with default
311 values
332 - seq_tile_size: `default=2048`, size of the kv tile size for attention
312 + seq_tile_size: `default=2048`, size of the kv tile size for attention
313 computation reduction
314
315 GQA support Notes:
336 - the spmd kernel for launching kernel should be on kv_heads instead of
316 + the spmd kernel for launching kernel should be on kv_heads instead of
317 nheads
318
319 Example usage:
@@ -415,18 +395,13 @@ def flash_paged_attention(
395 ), f"Need B_P_SIZE ({B_P_SIZE}) to be divisible by {block_size=}"
396 num_large_k_tile = context_kv_len // LARGE_TILE_SZ
397 num_blocks_per_large_tile = LARGE_TILE_SZ // block_size
418 - assert (num_blocks_per_large_tile <= B_P_SIZE
419 - ), f"The number of blocks in each large tile " \
420 - f"({num_blocks_per_large_tile}) shouldn't exceed partition size {B_P_SIZE}"
421 -
422 - block_tables_sbuf = nl.full((par_dim(B_P_SIZE), num_large_k_tile),
423 - 0,
424 - dtype=np.int32,
425 - buffer=nl.sbuf)
426 - for j in nl.affine_range(num_large_k_tile):
427 - i_p = nl.arange(num_blocks_per_large_tile)[:, None]
428 - block_tables_sbuf[i_p, j] = nl.load(
429 - block_tables[j * num_blocks_per_large_tile + i_p], dtype=np.int32)
398 + assert block_size % 32 == 0, "block_size is expected to be a multiple of 32"
399 + assert is_power_of_2(
400 + num_blocks_per_large_tile
401 + ), "The number of blocks in each large tile is expected of be power of 2"
402 + assert is_power_of_2(seqlen_q), "seqlen_q is expected to be power of 2"
403 +
404 + block_tables_sbuf = load_block_tables(block_tables, num_large_k_tile)
405
406 # Global Flash Attention accumulators
407 o_buffer = nl.zeros(
@@ -457,7 +432,7 @@ def flash_paged_attention(
432 )
433
434 for k_i in nl.affine_range(num_blocks_per_large_tile):
460 - loaded = nl.load(key_cache[block_tables_sbuf[k_i, j], :,
435 + loaded = nl.load(key_cache[block_tables_sbuf[j, k_i], :,
436 head_id, :])
437 cur_k_tile[:, nl.ds(k_i *
438 block_size, block_size)] = nl.transpose(loaded)
@@ -469,7 +444,7 @@ def flash_paged_attention(
444 num_blocks_per_partition):
445 v_i = (partition_idx * num_blocks_per_partition +
446 block_in_partition)
472 - loaded_v = nl.load(value_cache[block_tables_sbuf[v_i, j], :,
447 + loaded_v = nl.load(value_cache[block_tables_sbuf[j, v_i], :,
448 head_id, :])
449 cur_v_tile[
450 partition_idx,
@@ -477,14 +452,15 @@ def flash_paged_attention(
452 :,
453 ] = loaded_v
454
480 - cur_mask = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
481 - dtype=mask.dtype)
482 - for m_i in nl.affine_range(LARGE_TILE_SZ // B_F_SIZE):
483 - cur_mask[:, nl.ds(m_i * B_F_SIZE, B_F_SIZE)] = nl.load(
484 - mask[:, nl.ds(j * LARGE_TILE_SZ + m_i * B_F_SIZE, B_F_SIZE)])
485 -
486 - for i_q_h in nl.affine_range(q_h_per_k_h):
487 - for i in nl.affine_range(n_tile_q):
455 + for i in nl.affine_range(n_tile_q):
456 + cur_mask = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
457 + dtype=mask.dtype)
458 + for m_i in nl.affine_range(LARGE_TILE_SZ // B_F_SIZE):
459 + cur_mask[:, nl.ds(m_i * B_F_SIZE, B_F_SIZE)] = nl.load(mask[
460 + nl.ds(i * B_P_SIZE, B_P_SIZE),
461 + nl.ds(j * LARGE_TILE_SZ + m_i * B_F_SIZE, B_F_SIZE),
462 + ])
463 + for i_q_h in nl.affine_range(q_h_per_k_h):
464 q_tile = nl.ndarray((B_D_SIZE, B_P_SIZE), dtype=kernel_dtype)
465 q_hbm_tile = query[batch_id, head_id * q_h_per_k_h + i_q_h]
466 q_sbuf_tile = nl.load(
@@ -497,35 +473,24 @@ def flash_paged_attention(
473 q_local_tile=q_tile,
474 k=cur_k_tile,
475 v=cur_v_tile,
500 - q_h_per_k_h=q_h_per_k_h,
501 - seqlen_q=seqlen_q,
502 - nheads=h,
476 o_buffer=o_buffer[i, i_q_h],
477 l_buffer=l_buffer[:, i, i_q_h],
478 m_buffer=m_buffer[i, i_q_h],
506 - batch_id=batch_id,
507 - head_id=head_id,
508 - gqa_head_idx=i_q_h,
479 q_tile_idx=i,
510 - local_k_large_tile_idx=j,
480 kernel_dtype=kernel_dtype,
481 acc_type=acc_type,
482 flash_config=config,
483 use_causal_mask=False,
515 - continuous_batching_mask=cur_mask,
484 + tile_mask=cur_mask,
485 initialize=j == 0,
486 B_P_SIZE=B_P_SIZE,
487 B_F_SIZE=B_F_SIZE,
488 B_D_SIZE=B_D_SIZE,
520 - dropout_p=0.0,
521 - dropout_p_tensor=None,
522 - seed_tensor=None,
523 - logit_bias_tile=None,
489 )
490
491 # compute attention between input query, key and value
492 if key is not None and value is not None:
528 - B_F_SIZE = seqlen_q
493 + B_F_SIZE = min(seqlen_q, B_F_SIZE)
494 LARGE_TILE_SZ = seqlen_q
495 active_config = FlashConfig(
496 seq_tile_size=LARGE_TILE_SZ,
@@ -552,11 +517,16 @@ def flash_paged_attention(
517 config=active_config,
518 )
519
555 - cur_mask = nl.ndarray((par_dim(B_P_SIZE), B_F_SIZE), dtype=mask.dtype)
556 - cur_mask[:, :] = nl.load(mask[:, nl.ds(context_kv_len, B_F_SIZE)])
520 + for i in nl.affine_range(n_tile_q):
521 + cur_mask = nl.load(
522 + mask[
523 + nl.ds(i * B_P_SIZE, B_P_SIZE),
524 + nl.ds(context_kv_len, LARGE_TILE_SZ),
525 + ],
526 + dtype=mask.dtype,
527 + )
528 + for i_q_h in nl.affine_range(q_h_per_k_h):
529
558 - for i_q_h in nl.affine_range(q_h_per_k_h):
559 - for i in nl.affine_range(n_tile_q):
530 q_tile = nl.ndarray((B_D_SIZE, B_P_SIZE), dtype=kernel_dtype)
531 q_hbm_tile = query[batch_id, head_id * q_h_per_k_h + i_q_h]
532 q_sbuf_tile = nl.load(
@@ -568,32 +538,21 @@ def flash_paged_attention(
538 q_local_tile=q_tile,
539 k=cur_k_tile,
540 v=cur_v_tile,
571 - q_h_per_k_h=q_h_per_k_h,
572 - seqlen_q=seqlen_q,
573 - nheads=h,
541 o_buffer=o_buffer[i, i_q_h],
542 l_buffer=l_buffer[:, i, i_q_h],
543 m_buffer=m_buffer[i, i_q_h],
577 - batch_id=batch_id,
578 - head_id=head_id,
579 - gqa_head_idx=i_q_h,
544 q_tile_idx=i,
581 - local_k_large_tile_idx=0,
545 kernel_dtype=kernel_dtype,
546 acc_type=acc_type,
547 flash_config=active_config,
585 - use_causal_mask=False,
586 - continuous_batching_mask=cur_mask,
548 + use_causal_mask=True,
549 + tile_mask=cur_mask,
550 initialize=False,
551 B_P_SIZE=B_P_SIZE,
552 B_F_SIZE=B_F_SIZE,
553 B_D_SIZE=B_D_SIZE,
591 - dropout_p=0.0,
592 - dropout_p_tensor=None,
593 - seed_tensor=None,
594 - logit_bias_tile=None,
595 - qk_res_buffer=qk_res_buffer[i, i_q_h]
596 - if qk_res_buffer is not None else None,
554 + qk_res_buffer=(qk_res_buffer[i, i_q_h]
555 + if qk_res_buffer is not None else None),
556 )
557
558 # -- -- -- -- write output to buffer on HBM -- -- -- -- -- -- #
@@ -652,7 +611,6 @@ def flash_attn_varlen_nkifunc(
611 attn_mask,
612 n_kv_head=None,
613 head_size=None,
655 - B_P_SIZE=128,
614 LARGE_TILE_SZ=2048,
615 return_debug_tensors=False,
616 mixed_precision=True,