@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a1b3de86cd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Refactor the test code for attention kernels (#13)

Woosuk Kwon committed Mar 29, 2023 at 18:59 UTC a1b3de86cd6f27aeb299d45296a7409b8d2b7c0c
1 file changed +53 -19
tests/kernels/attention.py
+53 -19
@@ -1,5 +1,5 @@
1 import random
2 -from typing import Optional
2 +from typing import List, Optional
3
4 from flash_attn.flash_attention import FlashAttention
5 import torch
@@ -64,6 +64,39 @@ def ref_single_query_cached_kv_attention(
64 output[i].copy_(out, non_blocking=True)
65
66
67 +def ref_multi_query_kv_attention(
68 + cu_seq_lens: List[int],
69 + query: torch.Tensor,
70 + key: torch.Tensor,
71 + value: torch.Tensor,
72 + dtype: torch.dtype,
73 +) -> torch.Tensor:
74 + head_size = query.shape[-1]
75 + scale = 1.0 / (head_size ** 0.5)
76 +
77 + num_seqs = len(cu_seq_lens) - 1
78 + ref_outputs = []
79 + for i in range(num_seqs):
80 + start_idx = cu_seq_lens[i]
81 + end_idx = cu_seq_lens[i + 1]
82 + seq_len = end_idx - start_idx
83 +
84 + # Create attention mask
85 + attn_mask = torch.triu(torch.ones(seq_len, seq_len), diagonal=1) * -1e5
86 + attn_mask = attn_mask.to(dtype=dtype, device='cuda')
87 +
88 + ref_output = ref_masked_attention(
89 + query[start_idx:end_idx],
90 + key[start_idx:end_idx],
91 + value[start_idx:end_idx],
92 + scale,
93 + attn_mask=attn_mask,
94 + )
95 + ref_outputs.append(ref_output)
96 + ref_output = torch.cat(ref_outputs, dim=0)
97 + return ref_output
98 +
99 +
100 def test_single_query_cached_kv_attention(
101 num_tokens: int,
102 num_heads: int,
@@ -156,30 +189,29 @@ def test_multi_query_kv_attention(
189 causal=True,
190 )[0]
191
159 - ref_outputs = []
160 - for i, seq_len in enumerate(seq_lens):
161 - attn_mask = torch.triu(torch.ones(seq_len, seq_len), diagonal=1) * -1e5
162 - attn_mask = attn_mask.to(dtype=dtype, device='cuda')
163 - start_idx = cu_seq_lens[i]
164 - end_idx = cu_seq_lens[i + 1]
165 - ref_output = ref_masked_attention(
166 - query[start_idx:end_idx],
167 - key[start_idx:end_idx],
168 - value[start_idx:end_idx],
169 - scale,
170 - attn_mask=attn_mask,
171 - )
172 - ref_outputs.append(ref_output)
173 - ref_output = torch.cat(ref_outputs, dim=0)
174 -
192 + cu_seq_lens = cu_seq_lens.cpu().tolist()
193 + ref_output = ref_multi_query_kv_attention(
194 + cu_seq_lens,
195 + query,
196 + key,
197 + value,
198 + dtype,
199 + )
200 assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
201
202
203 @torch.inference_mode()
179 -def test_attention() -> None:
204 +def test_attention(seed: int) -> None:
205 + # NOTE(woosuk): Even when the seed is fixed, there is a chance that
206 + # the test fails due to the precision issue. Re-run the test if it fails.
207 + torch.random.manual_seed(seed)
208 + torch.cuda.manual_seed(seed)
209 for dtype in [torch.half, torch.float]:
210 for block_size in [8, 16]:
211 for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
212 + print(f'Testing single_query_cached_kv_attention with '
213 + f'dtype={dtype}, block_size={block_size}, '
214 + f'head_size={head_size}')
215 test_single_query_cached_kv_attention(
216 num_tokens=37,
217 num_heads=3,
@@ -193,6 +225,8 @@ def test_attention() -> None:
225 for dtype in [torch.half]:
226 # NOTE(woosuk): FlashAttention does not support head_size > 128.
227 for head_size in [64, 80, 96, 128]:
228 + print(f'Testing multi_query_kv_attention with dtype={dtype}, '
229 + f'head_size={head_size}')
230 test_multi_query_kv_attention(
231 num_seqs=11,
232 num_heads=3,
@@ -202,4 +236,4 @@ def test_attention() -> None:
236
237
238 if __name__ == '__main__':
205 - test_attention()
239 + test_attention(seed=0)