@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ddee88d0ff
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Neuron][Kernel] NKI-based flash-attention kernel with paged KV cache (#11277)

Signed-off-by: Liangfu Chen <liangfc@amazon.com> Co-authored-by: Jiangfei Duan <jfduan@outlook.com>

Liangfu Chen committed Jan 27, 2025 at 17:31 UTC ddee88d0ff2757bdef98a83a9c78af1ea4559758
3 files changed +1126 -1
.buildkite/run-neuron-test.sh
+1 -1
@@ -54,4 +54,4 @@ docker run --rm -it --device=/dev/neuron0 --device=/dev/neuron1 --network host \
54 -e "NEURON_COMPILE_CACHE_URL=${NEURON_COMPILE_CACHE_MOUNT}" \
55 --name "${container_name}" \
56 ${image_name} \
57 - /bin/bash -c "python3 /workspace/vllm/examples/offline_inference/neuron.py"
57 + /bin/bash -c "python3 /workspace/vllm/examples/offline_inference/neuron.py && python3 -m pytest /workspace/vllm/tests/neuron/ -v --capture=tee-sys"
tests/neuron/test_prefix_prefill.py new
+456
@@ -0,0 +1,456 @@
1 +import random
2 +from typing import Optional
3 +
4 +import pytest
5 +import torch
6 +import torch.nn.functional as F
7 +
8 +
9 +class BlockDiagonalCausalFromBottomRightMask:
10 +
11 + @staticmethod
12 + def _from_seqlens(query_lens, seq_lens, block_size=None):
13 + from torch import logical_and, logical_or
14 +
15 + contexted = block_size is None
16 + context_lens = torch.tensor(seq_lens) - torch.tensor(query_lens)
17 + n_queries = sum(query_lens)
18 + num_seqs = len(query_lens)
19 + if contexted:
20 + key_lens_blockaligned = seq_lens
21 + else:
22 + n_blocks_per_seq = (context_lens + block_size - 1) // block_size
23 + offset_per_seq = n_blocks_per_seq * block_size
24 + key_lens_blockaligned = offset_per_seq[:num_seqs].tolist()
25 + n_keys = sum(key_lens_blockaligned)
26 +
27 + a = (torch.arange(n_queries).reshape(n_queries,
28 + 1).expand(n_queries, n_keys))
29 + b = torch.arange(n_keys).reshape(1, n_keys).expand(n_queries, n_keys)
30 + q_cumsum = torch.tensor([0] + query_lens).cumsum(dim=0)
31 + k_cumsum = torch.tensor([0] + key_lens_blockaligned).cumsum(dim=0)
32 +
33 + prior_mask = torch.zeros(n_queries, n_keys)
34 + new_masks: list[torch.Tensor] = []
35 + for seq_id in range(num_seqs):
36 + ri = q_cumsum[seq_id]
37 + ci = k_cumsum[seq_id]
38 + nr = query_lens[seq_id]
39 +
40 + if contexted:
41 + nc = seq_lens[seq_id]
42 + a_offset = ci + nc - ri - nr
43 + new_mask = (a + a_offset) >= b
44 + else:
45 + nc = context_lens[seq_id]
46 + a_offset = ci + nc - 1
47 + new_mask = a_offset >= b
48 +
49 + left_mask = b >= ci
50 + top_mask = a >= ri
51 + bottom_mask = a < (ri + nr)
52 +
53 + new_mask = logical_and(
54 + logical_and(logical_and(new_mask, left_mask), top_mask),
55 + bottom_mask,
56 + )
57 + prior_mask = logical_or(prior_mask, new_mask)
58 + new_masks = new_masks + [new_mask]
59 + return prior_mask
60 +
61 + @staticmethod
62 + def from_seqlens(query_lens, seq_lens, block_size=None):
63 + contexted = block_size is None
64 + if contexted:
65 + prior_mask = BlockDiagonalCausalFromBottomRightMask._from_seqlens(
66 + query_lens, seq_lens)
67 + active_mask = None
68 + else:
69 + prior_mask = BlockDiagonalCausalFromBottomRightMask._from_seqlens(
70 + query_lens, seq_lens, block_size)
71 + active_mask = BlockDiagonalCausalFromBottomRightMask._from_seqlens(
72 + query_lens, query_lens)
73 + return prior_mask, active_mask
74 +
75 +
76 +def ref_softmax(x: torch.Tensor,
77 + dim: int,
78 + mixed_precision=False,
79 + return_max_reduce=False):
80 + max_value = torch.amax(x, dim=dim, keepdims=True)
81 + exp = torch.exp(x - max_value)
82 + if mixed_precision:
83 + sum_value = torch.sum(exp.astype(torch.float32),
84 + dim=dim,
85 + keepdims=True).astype(x.dtype)
86 + else:
87 + sum_value = torch.sum(exp, dim=dim, keepdims=True)
88 + if return_max_reduce:
89 + return exp / sum_value, max_value, torch.reciprocal(sum_value)
90 + return exp / sum_value
91 +
92 +
93 +def ref_masked_attention(
94 + query: torch.Tensor,
95 + key: torch.Tensor,
96 + value: torch.Tensor,
97 + scale: float,
98 + attn_mask: Optional[torch.Tensor] = None,
99 + return_max_reduce: Optional[bool] = False,
100 +) -> torch.Tensor:
101 + scaled_qk = scale * torch.einsum("qhd,khd->hqk", query, key).float()
102 + if attn_mask is not None:
103 + masked_score = scaled_qk + attn_mask.float()
104 + if return_max_reduce:
105 + norm_score, cached_max, cached_sum_reciprocal = ref_softmax(
106 + masked_score, dim=-1, return_max_reduce=True)
107 + else:
108 + norm_score = ref_softmax(masked_score, dim=-1)
109 + out = torch.einsum("hqk,khd->qhd", norm_score, value)
110 + if return_max_reduce:
111 + return (
112 + out,
113 + cached_max,
114 + cached_sum_reciprocal,
115 + norm_score,
116 + masked_score,
117 + scaled_qk,
118 + )
119 + else:
120 + return out
121 +
122 +
123 +def ref_context_attention(
124 + query,
125 + key,
126 + value,
127 + query_lens,
128 + seq_lens,
129 + head_size,
130 + num_kv_heads,
131 + num_heads,
132 + num_queries_per_kv,
133 + return_max_reduce=False,
134 +):
135 + scale = float(1.0 / (head_size**0.5))
136 + if num_queries_per_kv > 1:
137 + # Handle MQA and GQA
138 + key = torch.repeat_interleave(key, num_queries_per_kv, dim=1)
139 + value = torch.repeat_interleave(value, num_queries_per_kv, dim=1)
140 +
141 + attn_mask, _ = BlockDiagonalCausalFromBottomRightMask.from_seqlens(
142 + query_lens, seq_lens)
143 +
144 + # convert binary mask to -inf values
145 + attn_mask = torch.logical_not(attn_mask)
146 + attn_mask = attn_mask.float() * -30000
147 +
148 + output, cached_max, cached_sum_reciprocal, lse, masked_score, scaled_qk = (
149 + ref_masked_attention(
150 + query,
151 + key,
152 + value,
153 + scale,
154 + attn_mask,
155 + return_max_reduce=return_max_reduce,
156 + ))
157 +
158 + output = output.unsqueeze(1)
159 + if return_max_reduce:
160 + return (
161 + output,
162 + cached_max,
163 + cached_sum_reciprocal,
164 + lse,
165 + masked_score,
166 + scaled_qk,
167 + )
168 + else:
169 + return output
170 +
171 +
172 +@pytest.mark.parametrize(
173 + "num_heads,num_queries_per_kv,head_size,mixed_precision",
174 + [
175 + (4, 2, 8, False),
176 + (4, 2, 8, True),
177 + (32, 8, 64, True),
178 + ],
179 +)
180 +@torch.inference_mode()
181 +def test_contexted_kv_attention(
182 + num_heads: int,
183 + num_queries_per_kv: int,
184 + head_size: int,
185 + mixed_precision: bool,
186 +) -> None:
187 + import os
188 +
189 + import torch_xla.core.xla_model as xm
190 +
191 + from vllm.attention.ops.nki_flash_attn import flash_attn_varlen_nkifunc
192 +
193 + device = xm.xla_device()
194 +
195 + os.environ["NEURON_CC_FLAGS"] = (
196 + " --model-type=transformer -O1 "
197 + " --internal-hlo2tensorizer-options='--verify-hlo' ")
198 +
199 + random.seed(0)
200 + torch.manual_seed(0)
201 + torch.set_printoptions(sci_mode=False)
202 +
203 + min_ctx_len = 2
204 + max_ctx_len = 64
205 + min_query_len = 2
206 + max_query_len = 64
207 + prefill_batch_size = 2
208 + decode_batch_size = 6
209 + batch_size = prefill_batch_size + decode_batch_size
210 + block_size = 32
211 + max_model_len = (max_query_len + max_ctx_len) * 4
212 +
213 + max_block_per_request = max_model_len // block_size
214 + dtype = torch.float32
215 + cache_size = (batch_size * max_block_per_request) + 2
216 + ctx_lens = [
217 + random.randint(min_ctx_len, max_ctx_len)
218 + for _ in range(prefill_batch_size)
219 + ] + [
220 + random.randint(min_ctx_len, max_ctx_len)
221 + for _ in range(decode_batch_size)
222 + ]
223 + query_lens = [
224 + random.randint(min_query_len, max_query_len)
225 + for _ in range(prefill_batch_size)
226 + ] + [1 for _ in range(decode_batch_size)]
227 + seq_lens = [a + b for a, b in zip(query_lens, ctx_lens)]
228 + num_kv_heads = num_heads // num_queries_per_kv
229 +
230 + num_tokens = sum(query_lens)
231 + query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
232 + query.uniform_(-1, 1)
233 + torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
234 +
235 + kv = torch.empty(sum(seq_lens), 2, num_kv_heads, head_size, dtype=dtype)
236 + kv.uniform_(-1, 1)
237 + key, value = kv.unbind(dim=1)
238 +
239 + k_cache = torch.zeros(cache_size,
240 + block_size,
241 + num_kv_heads,
242 + head_size,
243 + dtype=dtype)
244 + v_cache = torch.zeros(cache_size,
245 + block_size,
246 + num_kv_heads,
247 + head_size,
248 + dtype=dtype)
249 + k = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
250 + v = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
251 + values = torch.arange(0, cache_size, dtype=torch.long)
252 + values = values[torch.randperm(cache_size)]
253 + block_table = values[:batch_size * max_block_per_request].view(
254 + batch_size, max_block_per_request)
255 + torch.tensor(seq_lens, dtype=torch.long)
256 + b_ctx_len = torch.tensor(ctx_lens, dtype=torch.long)
257 + b_start_loc = torch.cumsum(torch.tensor([0] + query_lens[:-1],
258 + dtype=torch.long),
259 + dim=0)
260 + # copy kv to cache
261 + b_seq_start_loc = torch.cumsum(torch.tensor([0] + seq_lens[:-1],
262 + dtype=torch.long),
263 + dim=0)
264 + for i in range(batch_size):
265 + for j in range(query_lens[i]):
266 + k[b_start_loc[i] + j].copy_(key[b_seq_start_loc[i] + b_ctx_len[i] +
267 + j])
268 + v[b_start_loc[i] + j].copy_(value[b_seq_start_loc[i] +
269 + b_ctx_len[i] + j])
270 + cur_ctx = 0
271 + block_id = 0
272 + while cur_ctx < b_ctx_len[i]:
273 + start_loc = b_seq_start_loc[i] + cur_ctx
274 + if cur_ctx + block_size > b_ctx_len[i]:
275 + end_loc = b_seq_start_loc[i] + b_ctx_len[i]
276 + else:
277 + end_loc = start_loc + block_size
278 + start_slot = block_table[i, block_id] * block_size
279 + end_slot = start_slot + end_loc - start_loc
280 + k_cache.view(-1, num_kv_heads,
281 + head_size)[start_slot:end_slot].copy_(
282 + key[start_loc:end_loc])
283 + v_cache.view(-1, num_kv_heads,
284 + head_size)[start_slot:end_slot].copy_(
285 + value[start_loc:end_loc])
286 + cur_ctx += block_size
287 + block_id += 1
288 +
289 + (
290 + output_ref,
291 + cached_max,
292 + cached_sum_reciprocal,
293 + lse,
294 + masked_score,
295 + scaled_qk,
296 + ) = ref_context_attention(
297 + query,
298 + key,
299 + value,
300 + query_lens,
301 + seq_lens,
302 + head_size,
303 + num_kv_heads,
304 + num_heads,
305 + num_queries_per_kv,
306 + return_max_reduce=True,
307 + )
308 +
309 + # build neuron program
310 + return_debug_tensors = False
311 + B_P_SIZE = 128
312 + LARGE_TILE_SZ = 2048
313 + max_num_queries = (
314 + (sum(query_lens) + block_size - 1) // block_size) * block_size
315 +
316 + def get_active_block_tables(block_tables, query_lens, seq_lens, block_size,
317 + num_blocks):
318 + context_lens = seq_lens - query_lens
319 + blocks_per_seq = (context_lens + block_size - 1) // block_size
320 + num_seqs = len(seq_lens)
321 + active_blocks: list[int] = []
322 + for seq_id in range(num_seqs):
323 + active_blocks = (
324 + active_blocks +
325 + block_tables[seq_id, :blocks_per_seq[seq_id]].tolist())
326 + return F.pad(
327 + torch.tensor(active_blocks),
328 + (0, num_blocks - len(active_blocks)),
329 + "constant",
330 + 0,
331 + )
332 +
333 + def shift_bit_length(x):
334 + return 1 << (x - 1).bit_length()
335 +
336 + # calculate input shapes
337 + max_num_queries_shifted = shift_bit_length(max_num_queries)
338 + max_num_queries_factor = B_P_SIZE // max_num_queries_shifted
339 + max_num_queries_padded = max_num_queries_shifted * max_num_queries_factor
340 + assert (max_num_queries_padded == B_P_SIZE
341 + ), "invalid {max_num_queries_padded=}"
342 + head_size_padded = B_P_SIZE
343 + context_lens = torch.tensor(seq_lens) - torch.tensor(query_lens)
344 + num_active_blocks_shifted = shift_bit_length(
345 + ((context_lens + block_size - 1) // block_size).sum().item())
346 + num_active_blocks_factor = (LARGE_TILE_SZ // block_size //
347 + num_active_blocks_shifted)
348 + num_active_blocks = num_active_blocks_shifted * num_active_blocks_factor
349 + assert (num_active_blocks *
350 + block_size) == LARGE_TILE_SZ, "invalid {num_active_blocks=}"
351 + context_kv_len = num_active_blocks * block_size
352 + assert context_kv_len == LARGE_TILE_SZ, f"invalid {context_kv_len=}"
353 +
354 + # pad QKV tensors
355 + pad_dims = (
356 + 0,
357 + head_size_padded - query.shape[2],
358 + 0,
359 + 0,
360 + 0,
361 + max_num_queries_padded - query.shape[0],
362 + )
363 + query = F.pad(query, pad_dims, "constant", 0)
364 + k = F.pad(k, pad_dims, "constant", 0)
365 + v = F.pad(v, pad_dims, "constant", 0)
366 + k_cache = F.pad(k_cache, (0, head_size_padded - head_size), "constant", 0)
367 + v_cache = F.pad(v_cache, (0, head_size_padded - head_size), "constant", 0)
368 +
369 + # permute QKV tensors
370 + # query: (1, n_heads, d, seq_q)
371 + # key: (1, n_kv_heads, d, seq_k)
372 + # value: (1, n_kv_heads, seq_v, d)
373 + query = query.unsqueeze(0).permute(0, 2, 3, 1).contiguous()
374 + k = k.unsqueeze(0).permute(0, 2, 3, 1).contiguous()
375 + v = v.unsqueeze(0).permute(0, 2, 1, 3).contiguous()
376 +
377 + # transform block table
378 + active_block_table = get_active_block_tables(
379 + block_table,
380 + torch.tensor(query_lens),
381 + torch.tensor(seq_lens),
382 + block_size,
383 + num_active_blocks,
384 + )
385 +
386 + # Build attention masks
387 + prior_mask, active_mask = (
388 + BlockDiagonalCausalFromBottomRightMask.from_seqlens(
389 + query_lens, seq_lens, block_size=block_size))
390 + attn_mask = torch.concat(
391 + [
392 + F.pad(
393 + prior_mask,
394 + (
395 + 0,
396 + context_kv_len - prior_mask.shape[1],
397 + 0,
398 + B_P_SIZE - prior_mask.shape[0],
399 + ),
400 + "constant",
401 + 0,
402 + ).bool(),
403 + F.pad(
404 + active_mask,
405 + (
406 + 0,
407 + B_P_SIZE - active_mask.shape[1],
408 + 0,
409 + B_P_SIZE - active_mask.shape[0],
410 + ),
411 + "constant",
412 + 0,
413 + ).bool(),
414 + ],
415 + dim=1,
416 + )
417 +
418 + input_args = (
419 + query.to(device=device),
420 + k.to(device=device),
421 + v.to(device=device),
422 + k_cache.to(device=device),
423 + v_cache.to(device=device),
424 + active_block_table.to(torch.int32).to(device=device),
425 + attn_mask.to(device=device),
426 + )
427 + input_kwargs = dict(
428 + n_kv_head=num_kv_heads,
429 + head_size=head_size,
430 + mixed_precision=mixed_precision,
431 + )
432 +
433 + if return_debug_tensors:
434 + output_nki, *debug_tensors = flash_attn_varlen_nkifunc(
435 + *input_args, **input_kwargs)
436 + else:
437 + output_nki = flash_attn_varlen_nkifunc(*input_args, **input_kwargs)
438 + debug_tensors = []
439 +
440 + output_nki = torch.tensor(output_nki).cpu()
441 + debug_tensors = [torch.tensor(dt).cpu() for dt in debug_tensors]
442 +
443 + num_actual_tokens = sum(query_lens)
444 + print(f"{num_actual_tokens=}")
445 + # - o: shape (bs, n_heads, seq_q, d) -> (bs, seq_q, n_heads, d)
446 + output_nki = output_nki.permute(
447 + 0, 2, 1, 3)[:, :, :, :head_size].cpu()[0, :num_actual_tokens, :, :]
448 + output_ref_padded = F.pad(
449 + output_ref,
450 + (0, 0, 0, 0, 0, 0, 0, max_num_queries_padded - output_ref.shape[0]),
451 + "constant",
452 + 0,
453 + )
454 + output_ref = output_ref_padded.transpose(0, 1)[0, :num_actual_tokens, :, :]
455 +
456 + torch.testing.assert_close(output_nki, output_ref, atol=1e-2, rtol=0)
vllm/attention/ops/nki_flash_attn.py new
+669
@@ -0,0 +1,669 @@
1 +from dataclasses import dataclass
2 +
3 +import neuronxcc.nki.isa as nisa
4 +import neuronxcc.nki.language as nl
5 +import numpy as np
6 +from neuronxcc import nki
7 +from neuronxcc.nki.language import par_dim
8 +
9 +
10 +@dataclass(frozen=True)
11 +class FlashConfig:
12 + """
13 + Config class for flash attention with default values
14 + """
15 +
16 + seq_tile_size: int = 2048
17 + should_transpose_v: bool = False
18 +
19 + __annotations__ = {
20 + "seq_tile_size": int,
21 + "should_transpose_v": bool,
22 + }
23 +
24 +
25 +@nki.jit
26 +def transpose_p_local(p_local_transposed,
27 + p_local,
28 + LARGE_TILE_SZ,
29 + forward_mask,
30 + B_F_SIZE=512):
31 + for i in nl.affine_range(LARGE_TILE_SZ // B_F_SIZE):
32 + if nisa.get_nc_version() == nisa.nc_version.gen3:
33 + p_local_t_tmp = nl.ndarray((par_dim(128), B_F_SIZE),
34 + buffer=nl.sbuf,
35 + dtype=p_local.dtype)
36 + else:
37 + p_local_t_tmp = nl.ndarray((par_dim(128), B_F_SIZE),
38 + buffer=nl.psum,
39 + dtype=np.float32)
40 +
41 + for j in nl.affine_range(B_F_SIZE // 128):
42 + j_128_slice = nl.ds(j * 128, 128)
43 + i_j_128_slice = nl.ds(i * B_F_SIZE + j * 128, 128)
44 +
45 + if nisa.get_nc_version() == nisa.nc_version.gen3:
46 + p_local_t_tmp[:, j_128_slice] = nisa.dma_transpose(
47 + p_local[:, i_j_128_slice], mask=forward_mask)
48 + else:
49 + p_local_t_tmp[:, j_128_slice] = nisa.nc_transpose(
50 + p_local[:, i_j_128_slice], mask=forward_mask)
51 +
52 + p_local_transposed[:, nl.ds(i * B_F_SIZE, B_F_SIZE)] = nl.copy(
53 + p_local_t_tmp, dtype=p_local_transposed.dtype, mask=forward_mask)
54 +
55 +
56 +@nki.jit
57 +def _flash_attention_core(
58 + q_local_tile,
59 + k,
60 + v,
61 + q_h_per_k_h,
62 + seqlen_q,
63 + nheads,
64 + o_buffer,
65 + l_buffer,
66 + m_buffer,
67 + batch_id,
68 + head_id,
69 + gqa_head_idx,
70 + q_tile_idx,
71 + local_k_large_tile_idx,
72 + kernel_dtype,
73 + acc_type,
74 + flash_config: FlashConfig,
75 + use_causal_mask=False,
76 + continuous_batching_mask=None,
77 + initialize=False,
78 + B_P_SIZE=128,
79 + B_F_SIZE=512,
80 + B_D_SIZE=128,
81 + dropout_p=0.0,
82 + dropout_p_tensor=None,
83 + seed_tensor=None,
84 + logit_bias_tile=None,
85 + qk_res_buffer=None,
86 +):
87 + """
88 + The flash attention core function to calculate self attention between a tile
89 + of q and a block of K and V.
90 + The q_local_tile has (B_P_SIZE, B_F_SIZE), which is loaded into the SBUF
91 + already. The block size of K and V
92 + is defined in the seq_tile_size of the flash_config. The results are stored
93 + in the following three buffers
94 + o_buffer: (B_P_SIZE, d)
95 + l_buffer: (B_P_SIZE, 1)
96 + m_buffer: (B_P_SIZE, 1)
97 + """
98 + LARGE_TILE_SZ = flash_config.seq_tile_size
99 + num_k_tile_per_large_tile = LARGE_TILE_SZ // B_F_SIZE
100 + seqlen_k = k.shape[-1]
101 + seqlen_q // B_P_SIZE
102 + seqlen_k // B_F_SIZE
103 +
104 + # TODO : support logit_bias with continuous_batching_mask
105 + assert not use_causal_mask, "causal mask is not supported."
106 + assert (continuous_batching_mask
107 + is not None), "continuous_batching_mask input is required."
108 + if continuous_batching_mask is not None:
109 + assert (logit_bias_tile is
110 + None), "continuous_batching_mask does not support logit_bias!"
111 +
112 + # mask are used to only apply computation to the lower half of the matrix,
113 + # which reduce the arthimetic intensity by half
114 + forward_mask = (q_tile_idx * B_P_SIZE >= local_k_large_tile_idx *
115 + LARGE_TILE_SZ if use_causal_mask else None)
116 +
117 + qk_res_buf = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
118 + buffer=nl.sbuf,
119 + dtype=acc_type)
120 + max_local = nl.ndarray((par_dim(B_P_SIZE), num_k_tile_per_large_tile),
121 + dtype=acc_type)
122 + for k_i in nl.affine_range(num_k_tile_per_large_tile):
123 + k_i_b_f_slice = nl.ds(k_i * B_F_SIZE, B_F_SIZE)
124 +
125 + qk_psum = nl.zeros((par_dim(B_P_SIZE), B_F_SIZE),
126 + dtype=np.float32,
127 + buffer=nl.psum) # (128, 512)
128 + qk_psum[:, :] = nl.matmul(q_local_tile,
129 + k[:, k_i_b_f_slice],
130 + transpose_x=True,
131 + mask=None) # (p(128), 512)
132 +
133 + qk_res_buf[:, k_i_b_f_slice] = nl.where(
134 + continuous_batching_mask[:, k_i_b_f_slice],
135 + qk_psum[:, nl.ds(0, B_F_SIZE)],
136 + -9984.0,
137 + dtype=acc_type,
138 + )
139 +
140 + # Calculate max of the current tile
141 + max_local[:, k_i] = nisa.tensor_reduce(
142 + np.max,
143 + qk_res_buf[:, k_i_b_f_slice],
144 + axis=(1, ),
145 + dtype=acc_type,
146 + negate=False,
147 + mask=forward_mask,
148 + )
149 +
150 + if qk_res_buffer is not None:
151 + qk_res_buffer[:, :] = nl.copy(qk_res_buf[:, :])
152 +
153 + max_ = nisa.tensor_reduce(
154 + np.max,
155 + max_local[:, :],
156 + axis=(1, ),
157 + dtype=acc_type,
158 + negate=False,
159 + mask=forward_mask,
160 + )
161 +
162 + o_previous_scaled = nl.ndarray((par_dim(B_P_SIZE), B_D_SIZE),
163 + dtype=o_buffer.dtype)
164 +
165 + if initialize:
166 + m_buffer[:, 0] = nl.copy(max_)
167 + m_current = max_
168 + else:
169 + m_previous = nl.copy(m_buffer[:, 0])
170 + m_buffer[:, 0] = nl.maximum(m_previous, max_,
171 + mask=forward_mask) # (128,1)
172 +
173 + m_current = m_buffer[:, 0]
174 + # Compute scaling factor
175 + alpha = nisa.activation(
176 + np.exp,
177 + m_previous,
178 + bias=-1 * m_current,
179 + scale=1.0,
180 + mask=forward_mask,
181 + )
182 + o_previous_scaled[...] = nl.multiply(o_buffer[:, :],
183 + alpha,
184 + mask=forward_mask)
185 +
186 + p_local = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
187 + dtype=kernel_dtype)
188 + REDUCTION_TILE = min(2048, LARGE_TILE_SZ // 2)
189 +
190 + p_partial_sum = nl.ndarray(
191 + (par_dim(B_P_SIZE), LARGE_TILE_SZ // REDUCTION_TILE), dtype=acc_type)
192 +
193 + for k_r_i in nl.affine_range(LARGE_TILE_SZ // REDUCTION_TILE):
194 + k_r_i_reduce_slice = nl.ds(k_r_i * REDUCTION_TILE, REDUCTION_TILE)
195 +
196 + # compute exp(qk - max)
197 + # Compute partial row - tile sum of exp(qk - max))
198 + # FIXME : Use activation accumulate to accumulate over k_r_i loop ?
199 + p_local[:, k_r_i_reduce_slice] = nisa.activation_reduce(
200 + np.exp,
201 + qk_res_buf[:, k_r_i_reduce_slice],
202 + bias=-1 * m_current,
203 + scale=1.0,
204 + reduce_op=nl.add,
205 + reduce_res=p_partial_sum[:, k_r_i],
206 + dtype=kernel_dtype,
207 + mask=forward_mask,
208 + )
209 +
210 + ps = nl.sum(p_partial_sum, axis=1, dtype=acc_type, mask=forward_mask)
211 +
212 + p_local_transposed = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
213 + dtype=kernel_dtype)
214 + transpose_p_local(
215 + p_local_transposed=p_local_transposed,
216 + p_local=p_local,
217 + LARGE_TILE_SZ=LARGE_TILE_SZ,
218 + forward_mask=forward_mask,
219 + B_F_SIZE=B_F_SIZE,
220 + )
221 +
222 + pv_psum = nl.zeros((par_dim(B_P_SIZE), B_D_SIZE),
223 + dtype=np.float32,
224 + buffer=nl.psum)
225 + for k_i in nl.affine_range(LARGE_TILE_SZ // B_P_SIZE):
226 + pv_psum[:, :] += nl.matmul(
227 + p_local_transposed[:, nl.ds(k_i * B_P_SIZE, B_P_SIZE)],
228 + v[k_i, :, :],
229 + transpose_x=True,
230 + mask=forward_mask,
231 + ) # (128, 128) (p(Br), d)
232 +
233 + if initialize:
234 + o_buffer[:, :] = nl.copy(pv_psum[:, :])
235 + l_buffer[:, 0] = nl.add(nl.log(ps), max_)
236 + else:
237 + o_buffer[:, :] = nl.add(o_previous_scaled, pv_psum, mask=forward_mask)
238 +
239 + l_prev = l_buffer[:, 0]
240 + l_exp = nl.add(
241 + nl.exp(
242 + nl.subtract(l_prev, m_current, mask=forward_mask),
243 + mask=forward_mask,
244 + ),
245 + ps,
246 + mask=forward_mask,
247 + )
248 + l_buffer[:, 0] = nl.add(m_current,
249 + nl.log(l_exp, mask=forward_mask),
250 + mask=forward_mask)
251 +
252 +
253 +@nki.jit
254 +def load_v_tile(v_hbm_tile, cur_v_tile, j, v_i, config):
255 + LARGE_TILE_SZ = config.seq_tile_size
256 + B_P_SIZE = 128
257 +
258 + if not config.should_transpose_v:
259 + cur_v_tile[v_i, :, :] = nl.load(
260 + v_hbm_tile[nl.ds(j * LARGE_TILE_SZ + B_P_SIZE * v_i, B_P_SIZE), :],
261 + dtype=cur_v_tile.dtype,
262 + )
263 + return
264 +
265 + if nisa.get_nc_version() == nisa.nc_version.gen3:
266 + cur_v_tile_transposed = nisa.dma_transpose(
267 + v_hbm_tile[:,
268 + nl.ds(j * LARGE_TILE_SZ + B_P_SIZE * v_i, B_P_SIZE)])
269 + cur_v_tile[v_i, :, :] = nisa.tensor_copy(cur_v_tile_transposed,
270 + dtype=cur_v_tile.dtype)
271 + return
272 +
273 + cur_v_tile[v_i, :, :] = nl.load_transpose2d(
274 + v_hbm_tile[:, nl.ds(j * LARGE_TILE_SZ + B_P_SIZE * v_i, B_P_SIZE)],
275 + dtype=cur_v_tile.dtype,
276 + )
277 +
278 +
279 +@nki.jit
280 +def flash_paged_attention(
281 + query,
282 + key,
283 + value,
284 + key_cache,
285 + value_cache,
286 + block_tables,
287 + mask,
288 + softmax_scale=None,
289 + mixed_precision=True,
290 + config=None,
291 + return_debug_tensors=False,
292 +):
293 + """
294 + Flash PagedAttention Forward Kernel.
295 + - PagedAttention Paper: https://arxiv.org/abs/2309.06180
296 + - Chunked Prefill Paper: https://arxiv.org/abs/2403.02310
297 +
298 + IO tensor layouts:
299 + - query: shape (1, n_heads, d, seq_q)
300 + - key: shape (1, n_kv_heads, d, seq_k)
301 + - value: shape (1, n_kv_heads, seq_v, d)
302 + - key_cache: (num_blocks, block_size, n_kv_heads, d)
303 + - value_cache: (num_blocks, block_size, n_kv_heads, d)
304 + - block_tables: (num_active_blocks, )
305 + - mask: (seq_q, num_active_blocks * block_size)
306 + - o: shape (1, n_heads, seq_q, d)
307 + - l_m: shape (1, n_heads, seq_q, 2)
308 +
309 + - This kernel requires seq_k == seq_v
310 + - We use continuous batching by default, so the batch dimension is
311 + always 1, and different requests are concatenated along sequence
312 + dimension.
313 + - We use paged cache blocks (key_cache, value_cache) to store KV cache.
314 +
315 + IO tensor dtypes:
316 + - This kernel assumes all IO tensors have the same dtype except for
317 + block_tables (int32) and mask (int32)
318 + - If mixed_percision is True, then all Tensor Engine operation will be
319 + performed in bfloat16 and accumulation will be performed in float32.
320 + Otherwise the intermediates will be in the same type as the inputs.
321 +
322 + Compile-time Constants:
323 + - softmax_scale: scaling for softmax, is None, default is `1.0/(d**0.5)`
324 + - mixed_precision: flag to set non-matmul ops in fp32 precision, default
325 + is set to `true`, if false, we use same precision as input types
326 + - config: Instance of dataclass :class:`nki.kernels.attention.FlashConfig`
327 + with Performance config parameters for flash attention with default
328 + values
329 + seq_tile_size: `default=2048`, size of the kv tile size for attention
330 + computation reduction
331 +
332 + GQA support Notes:
333 + the spmd kernel for launching kernel should be on kv_heads instead of
334 + nheads
335 +
336 + Example usage:
337 + MHA: q: [b, h, d, s], k: [b, h, d, s], v: [b, h, s, d]
338 + usage: `flash_fwd[b, h](q, k, v, ...)`
339 + GQA: q: [b, h, d, s], k: [b, kv_h, d, s], v: [b, kv_h, s, d]
340 + usage: `flash_fwd[b, kv_h](q, k, v, ...)`
341 + """
342 + config = config or FlashConfig()
343 + B_F_SIZE = 512
344 + B_P_SIZE = 128
345 + b, h, d, seqlen_q = query.shape
346 + B_D_SIZE = d
347 + LARGE_TILE_SZ = config.seq_tile_size
348 + n_tile_q = seqlen_q // B_P_SIZE # since q will be loaded on tensor engine
349 + num_blocks, block_size, k_h, _ = key_cache.shape
350 + q_h_per_k_h = h // k_h
351 + assert tuple(key_cache.shape) == (
352 + num_blocks,
353 + block_size,
354 + k_h,
355 + d,
356 + ), "Input shape mismatch!"
357 + assert tuple(value_cache.shape) == (
358 + num_blocks,
359 + block_size,
360 + k_h,
361 + d,
362 + ), "Input shape mismatch!"
363 + assert b == 1, f"invalid batch size {b=}"
364 + assert d <= 128, f" we do not support head_dim > 128, got head dim {d}"
365 + kernel_dtype = nl.bfloat16 if mixed_precision else query.dtype
366 + acc_type = np.dtype(np.float32) if mixed_precision else kernel_dtype
367 +
368 + o = nl.ndarray((b, h, seqlen_q, d),
369 + dtype=query.dtype,
370 + buffer=nl.shared_hbm)
371 + hbm_l_buffer, hbm_m_buffer, hbm_qk_res, qk_res_buffer = (
372 + None,
373 + None,
374 + None,
375 + None,
376 + )
377 + if return_debug_tensors:
378 + hbm_l_buffer = nl.ndarray((b, h, seqlen_q),
379 + dtype=acc_type,
380 + buffer=nl.shared_hbm)
381 + hbm_m_buffer = nl.ndarray((b, h, seqlen_q),
382 + dtype=acc_type,
383 + buffer=nl.shared_hbm)
384 + hbm_qk_res = nl.ndarray((b, h, B_P_SIZE, seqlen_q),
385 + dtype=acc_type,
386 + buffer=nl.shared_hbm)
387 + qk_res_buffer = nl.zeros(
388 + (n_tile_q, q_h_per_k_h, par_dim(B_P_SIZE), seqlen_q),
389 + dtype=acc_type,
390 + buffer=nl.sbuf,
391 + lazy_initialization=True,
392 + )
393 +
394 + assert (
395 + nl.program_ndim() == 2
396 + ), f"Expect spmd grid with 2 dimensions, got {nl.program_ndim()} instead!"
397 + batch_id = nl.program_id(axis=0)
398 + head_id = nl.program_id(axis=1)
399 +
400 + softmax_scale = softmax_scale or (1.0 / (d**0.5))
401 +
402 + (num_active_blocks, ) = block_tables.shape
403 + context_kv_len = num_active_blocks * block_size
404 + assert (config.seq_tile_size >= 512
405 + ), f" seq tile_size {config.seq_tile_size} cannot be less than 512"
406 + assert (context_kv_len % LARGE_TILE_SZ == 0
407 + ), f"Need {context_kv_len=} to be divisible by {LARGE_TILE_SZ=}"
408 + assert (
409 + LARGE_TILE_SZ % B_P_SIZE == 0
410 + ), f"Need LARGE_TILE_SZ ({LARGE_TILE_SZ}) to be divisible by {B_P_SIZE=}"
411 + assert (B_P_SIZE % block_size == 0
412 + ), f"Need B_P_SIZE ({B_P_SIZE}) to be divisible by {block_size=}"
413 + num_large_k_tile = context_kv_len // LARGE_TILE_SZ
414 + num_blocks_per_large_tile = LARGE_TILE_SZ // block_size
415 + assert (num_blocks_per_large_tile <= B_P_SIZE
416 + ), f"The number of blocks in each large tile " \
417 + f"({num_blocks_per_large_tile}) shouldn't exceed partition size {B_P_SIZE}"
418 +
419 + block_tables_sbuf = nl.full((par_dim(B_P_SIZE), num_large_k_tile),
420 + 0,
421 + dtype=np.int32,
422 + buffer=nl.sbuf)
423 + for j in nl.affine_range(num_large_k_tile):
424 + i_p = nl.arange(num_blocks_per_large_tile)[:, None]
425 + block_tables_sbuf[i_p, j] = nl.load(
426 + block_tables[j * num_blocks_per_large_tile + i_p], dtype=np.int32)
427 +
428 + # Global Flash Attention accumulators
429 + o_buffer = nl.zeros(
430 + (n_tile_q, q_h_per_k_h, par_dim(B_P_SIZE), d),
431 + dtype=acc_type,
432 + buffer=nl.sbuf,
433 + lazy_initialization=True,
434 + )
435 + l_buffer = nl.zeros(
436 + (par_dim(B_P_SIZE), n_tile_q, q_h_per_k_h),
437 + dtype=acc_type,
438 + buffer=nl.sbuf,
439 + lazy_initialization=True,
440 + )
441 + m_buffer = nl.zeros(
442 + (n_tile_q, q_h_per_k_h, par_dim(B_P_SIZE), 1),
443 + dtype=acc_type,
444 + buffer=nl.sbuf,
445 + lazy_initialization=True,
446 + )
447 +
448 + for j in nl.sequential_range(0, num_large_k_tile):
449 + cur_k_tile = nl.ndarray((par_dim(B_D_SIZE), LARGE_TILE_SZ),
450 + dtype=kernel_dtype)
451 + cur_v_tile = nl.ndarray(
452 + (LARGE_TILE_SZ // B_P_SIZE, par_dim(B_P_SIZE), B_D_SIZE),
453 + dtype=kernel_dtype,
454 + )
455 +
456 + for k_i in nl.affine_range(num_blocks_per_large_tile):
457 + loaded = nl.load(key_cache[block_tables_sbuf[k_i, j], :,
458 + head_id, :])
459 + cur_k_tile[:, nl.ds(k_i *
460 + block_size, block_size)] = nl.transpose(loaded)
461 +
462 + load_tile_size = B_P_SIZE
463 + num_blocks_per_partition = load_tile_size // block_size
464 + for partition_idx in nl.affine_range(LARGE_TILE_SZ // load_tile_size):
465 + for block_in_partition in nl.affine_range(
466 + num_blocks_per_partition):
467 + v_i = (partition_idx * num_blocks_per_partition +
468 + block_in_partition)
469 + loaded_v = nl.load(value_cache[block_tables_sbuf[v_i, j], :,
470 + head_id, :])
471 + cur_v_tile[partition_idx,
472 + nl.ds(block_in_partition *
473 + block_size, block_size), :, ] = loaded_v
474 +
475 + cur_mask = nl.ndarray((par_dim(B_P_SIZE), LARGE_TILE_SZ),
476 + dtype=mask.dtype)
477 + for m_i in nl.affine_range(LARGE_TILE_SZ // B_F_SIZE):
478 + cur_mask[:, nl.ds(m_i * B_F_SIZE, B_F_SIZE)] = nl.load(
479 + mask[:, nl.ds(j * LARGE_TILE_SZ + m_i * B_F_SIZE, B_F_SIZE)])
480 +
481 + for i_q_h in nl.affine_range(q_h_per_k_h):
482 + for i in nl.affine_range(n_tile_q):
483 + q_tile = nl.ndarray((B_D_SIZE, B_P_SIZE), dtype=kernel_dtype)
484 + q_hbm_tile = query[batch_id, head_id * q_h_per_k_h + i_q_h]
485 + q_sbuf_tile = nl.load(
486 + q_hbm_tile[:, nl.ds(i * B_P_SIZE, B_P_SIZE)],
487 + dtype=kernel_dtype,
488 + ) # load (d, 128) tile in SBUF
489 + q_tile[:, :] = q_sbuf_tile * softmax_scale
490 +
491 + _flash_attention_core(
492 + q_local_tile=q_tile,
493 + k=cur_k_tile,
494 + v=cur_v_tile,
495 + q_h_per_k_h=q_h_per_k_h,
496 + seqlen_q=seqlen_q,
497 + nheads=h,
498 + o_buffer=o_buffer[i, i_q_h],
499 + l_buffer=l_buffer[:, i, i_q_h],
500 + m_buffer=m_buffer[i, i_q_h],
501 + batch_id=batch_id,
502 + head_id=head_id,
503 + gqa_head_idx=i_q_h,
504 + q_tile_idx=i,
505 + local_k_large_tile_idx=j,
506 + kernel_dtype=kernel_dtype,
507 + acc_type=acc_type,
508 + flash_config=config,
509 + use_causal_mask=False,
510 + continuous_batching_mask=cur_mask,
511 + initialize=j == 0,
512 + B_P_SIZE=B_P_SIZE,
513 + B_F_SIZE=B_F_SIZE,
514 + B_D_SIZE=B_D_SIZE,
515 + dropout_p=0.0,
516 + dropout_p_tensor=None,
517 + seed_tensor=None,
518 + logit_bias_tile=None,
519 + )
520 +
521 + # compute attention between input query, key and value
522 + if key is not None and value is not None:
523 + B_F_SIZE = seqlen_q
524 + LARGE_TILE_SZ = seqlen_q
525 + active_config = FlashConfig(
526 + seq_tile_size=LARGE_TILE_SZ,
527 + should_transpose_v=config.should_transpose_v,
528 + )
529 +
530 + cur_k_tile = nl.ndarray((par_dim(B_D_SIZE), LARGE_TILE_SZ),
531 + dtype=kernel_dtype)
532 + cur_v_tile = nl.ndarray(
533 + (LARGE_TILE_SZ // B_P_SIZE, par_dim(B_P_SIZE), B_D_SIZE),
534 + dtype=kernel_dtype,
535 + )
536 +
537 + cur_k_tile[:, :] = nl.load(key[batch_id, head_id, :, :])
538 +
539 + load_tile_size = B_P_SIZE
540 + v_hbm_tile = value[batch_id, head_id]
541 + for v_i in nl.affine_range(LARGE_TILE_SZ // load_tile_size):
542 + load_v_tile(
543 + v_hbm_tile=v_hbm_tile,
544 + cur_v_tile=cur_v_tile,
545 + j=0,
546 + v_i=v_i,
547 + config=active_config,
548 + )
549 +
550 + cur_mask = nl.ndarray((par_dim(B_P_SIZE), B_F_SIZE), dtype=mask.dtype)
551 + cur_mask[:, :] = nl.load(mask[:, nl.ds(context_kv_len, B_F_SIZE)])
552 +
553 + for i_q_h in nl.affine_range(q_h_per_k_h):
554 + for i in nl.affine_range(n_tile_q):
555 + q_tile = nl.ndarray((B_D_SIZE, B_P_SIZE), dtype=kernel_dtype)
556 + q_hbm_tile = query[batch_id, head_id * q_h_per_k_h + i_q_h]
557 + q_sbuf_tile = nl.load(
558 + q_hbm_tile[:, nl.ds(i * B_P_SIZE, B_P_SIZE)],
559 + dtype=kernel_dtype,
560 + ) # load (d, 128) tile in SBUF
561 + q_tile[:, :] = q_sbuf_tile * softmax_scale
562 + _flash_attention_core(
563 + q_local_tile=q_tile,
564 + k=cur_k_tile,
565 + v=cur_v_tile,
566 + q_h_per_k_h=q_h_per_k_h,
567 + seqlen_q=seqlen_q,
568 + nheads=h,
569 + o_buffer=o_buffer[i, i_q_h],
570 + l_buffer=l_buffer[:, i, i_q_h],
571 + m_buffer=m_buffer[i, i_q_h],
572 + batch_id=batch_id,
573 + head_id=head_id,
574 + gqa_head_idx=i_q_h,
575 + q_tile_idx=i,
576 + local_k_large_tile_idx=0,
577 + kernel_dtype=kernel_dtype,
578 + acc_type=acc_type,
579 + flash_config=active_config,
580 + use_causal_mask=False,
581 + continuous_batching_mask=cur_mask,
582 + initialize=False,
583 + B_P_SIZE=B_P_SIZE,
584 + B_F_SIZE=B_F_SIZE,
585 + B_D_SIZE=B_D_SIZE,
586 + dropout_p=0.0,
587 + dropout_p_tensor=None,
588 + seed_tensor=None,
589 + logit_bias_tile=None,
590 + qk_res_buffer=qk_res_buffer[i, i_q_h]
591 + if qk_res_buffer is not None else None,
592 + )
593 +
594 + # -- -- -- -- write output to buffer on HBM -- -- -- -- -- -- #
595 + for i_q_h in nl.affine_range(q_h_per_k_h):
596 + for i in nl.affine_range(n_tile_q):
597 + out = nl.multiply(
598 + o_buffer[i, i_q_h, :, :],
599 + nl.exp(m_buffer[i, i_q_h, :, :] - l_buffer[:, i, i_q_h]),
600 + dtype=kernel_dtype,
601 + )
602 +
603 + nl.store(
604 + o[batch_id, head_id * q_h_per_k_h + i_q_h,
605 + nl.ds(i * B_P_SIZE, B_P_SIZE), :, ],
606 + out,
607 + )
608 + # maximum and summation statistics
609 + if return_debug_tensors:
610 + nl.store(
611 + hbm_m_buffer[batch_id, head_id * q_h_per_k_h + i_q_h,
612 + nl.ds(i * B_P_SIZE, B_P_SIZE), ],
613 + m_buffer[i, i_q_h, :, :],
614 + )
615 + nl.store(
616 + hbm_l_buffer[batch_id, head_id * q_h_per_k_h + i_q_h,
617 + nl.ds(i * B_P_SIZE, B_P_SIZE), ],
618 + l_buffer[:, i, i_q_h],
619 + )
620 + nl.store(
621 + hbm_qk_res[batch_id, head_id * q_h_per_k_h + i_q_h, :, :],
622 + qk_res_buffer[batch_id, i_q_h, :, :],
623 + )
624 +
625 + if return_debug_tensors:
626 + return o, hbm_m_buffer, hbm_l_buffer, hbm_qk_res
627 + return o
628 +
629 +
630 +def flash_attn_varlen_nkifunc(
631 + query,
632 + key,
633 + value,
634 + key_cache,
635 + value_cache,
636 + block_table,
637 + attn_mask,
638 + n_kv_head=None,
639 + head_size=None,
640 + B_P_SIZE=128,
641 + LARGE_TILE_SZ=2048,
642 + return_debug_tensors=False,
643 + mixed_precision=True,
644 +):
645 + config = FlashConfig(
646 + seq_tile_size=LARGE_TILE_SZ,
647 + should_transpose_v=False,
648 + )
649 + kwargs = dict(
650 + query=query,
651 + key=key,
652 + value=value,
653 + key_cache=key_cache,
654 + value_cache=value_cache,
655 + block_tables=block_table,
656 + mask=attn_mask,
657 + softmax_scale=1.0 / (head_size**0.5),
658 + config=config,
659 + mixed_precision=mixed_precision,
660 + return_debug_tensors=return_debug_tensors,
661 + )
662 + _, n_kv_head, _, _ = key.shape
663 +
664 + if return_debug_tensors:
665 + o, *debug_tensors = flash_paged_attention[1, n_kv_head](**kwargs)
666 + return o, *debug_tensors
667 + else:
668 + o = flash_paged_attention[1, n_kv_head](**kwargs)
669 + return o