A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add query stride to multi_query_cached_kv_attention & Add kernel benchmark script (#27)
* Add query stride to multi_query_cached_kv_attention * Add kernel benchmark script
Woosuk Kwon committed
Apr 8, 2023 at 13:36 UTC
c267b1a02c952b68a897c96201f32ad57e0b955e
3 files changed
+181
-8
benchmark/benchmark_attention.py
new
+165
@@ -0,0 +1,165 @@
1
+import functools
2
+import random
3
+import time
4
+from typing import List
5
+
6
+from flash_attn.flash_attn_interface import _flash_attn_forward
7
+import torch
8
+
9
+from cacheflow import attention_ops
10
+
11
+
12
+def benchmark(name, f, num_warmup = 10, num_iters = 100):
13
+ for _ in range(num_warmup):
14
+ f()
15
+ torch.cuda.synchronize()
16
+
17
+ start = time.time()
18
+ for _ in range(num_iters):
19
+ f()
20
+ torch.cuda.synchronize()
21
+ end = time.time()
22
+ print(f'{name}: {(end - start) / num_iters * 1000:.3f} ms')
23
+
24
+
25
+@torch.inference_mode()
26
+def benchmark_multi_query_cached_kv_attention(
27
+ query_lens: List[int],
28
+ context_lens: List[int],
29
+ num_heads: int,
30
+ head_size: int,
31
+ block_size: int,
32
+ num_blocks: int,
33
+ dtype: torch.dtype,
34
+) -> None:
35
+ print(f'query_lens: {query_lens}, context_lens: {context_lens}, '
36
+ f'num_heads: {num_heads}, head_size: {head_size}, block_size: '
37
+ f'{block_size}, num_blocks: {num_blocks}, dtype: {dtype}')
38
+ # Create query tensor.
39
+ num_queries = len(query_lens)
40
+ cu_query_lens = [0]
41
+ for query_len in query_lens:
42
+ cu_query_lens.append(cu_query_lens[-1] + query_len)
43
+ num_total_tokens = cu_query_lens[-1]
44
+ qkv = torch.randn(
45
+ num_total_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
46
+ query, _, _ = qkv.unbind(dim=1)
47
+
48
+ # Create key and value cache.
49
+ x = 16 // torch.tensor([], dtype=dtype).element_size()
50
+ key_block_shape = (num_heads, head_size // x, block_size, x)
51
+ key_cache = torch.randn(
52
+ size=(num_blocks, *key_block_shape), dtype=dtype, device='cuda')
53
+ value_block_shape = (num_heads, head_size, block_size)
54
+ value_cache = torch.randn(
55
+ size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
56
+
57
+ # Create block tables.
58
+ max_context_len = max(context_lens)
59
+ max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
60
+ block_tables = []
61
+ for _ in range(num_queries):
62
+ block_table = [
63
+ random.randint(0, num_blocks - 1)
64
+ for _ in range(max_num_blocks_per_seq)
65
+ ]
66
+ block_tables.append(block_table)
67
+ block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
68
+
69
+ # Create input and output data structures.
70
+ cu_query_lens = torch.tensor(cu_query_lens, dtype=torch.int, device='cuda')
71
+ context_len_tensor = torch.tensor(context_lens, dtype=torch.int, device='cuda')
72
+ scale = float(1.0 / (head_size ** 0.5))
73
+ output = torch.empty(
74
+ num_total_tokens, num_heads, head_size, dtype=dtype, device='cuda')
75
+
76
+ # Run our implementation.
77
+ def run_ours():
78
+ attention_ops.multi_query_cached_kv_attention(
79
+ cu_query_lens,
80
+ output,
81
+ query,
82
+ key_cache,
83
+ value_cache,
84
+ scale,
85
+ block_tables,
86
+ context_len_tensor,
87
+ block_size,
88
+ max_context_len,
89
+ )
90
+ benchmark('Ours', run_ours)
91
+
92
+ # Upper bound: Flash attention.
93
+ # Becuase Flash attention cannot read our own cache,
94
+ # we make key and value tensors contiguous.
95
+ num_kv_tokens = sum(context_lens)
96
+ cu_context_lens = [0]
97
+ for context_len in context_lens:
98
+ cu_context_lens.append(cu_context_lens[-1] + context_len)
99
+ cu_context_lens = torch.tensor(cu_context_lens, dtype=torch.int, device='cuda')
100
+ qkv = torch.randn(
101
+ num_kv_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
102
+ _, key, value = qkv.unbind(dim=1)
103
+ ref_output = torch.empty_like(output)
104
+
105
+ # Run Flash attention.
106
+ def run_flash_attn():
107
+ _flash_attn_forward(
108
+ query,
109
+ key,
110
+ value,
111
+ ref_output,
112
+ cu_query_lens,
113
+ cu_context_lens,
114
+ max(query_lens),
115
+ max_context_len,
116
+ dropout_p=0.0,
117
+ softmax_scale=scale,
118
+ causal=True,
119
+ return_softmax=False,
120
+ )
121
+ benchmark('Flash attention', run_flash_attn)
122
+
123
+
124
+if __name__ == '__main__':
125
+ BLOCK_SIZE = 8
126
+ NUM_BLOCKS = 1024
127
+ DTYPE = torch.half
128
+
129
+ # LLaMA-13B and OPT-13B
130
+ NUM_HEADS = 40
131
+ HEAD_SIZE = 128
132
+
133
+ run_benchmark = functools.partial(
134
+ benchmark_multi_query_cached_kv_attention,
135
+ num_heads=NUM_HEADS,
136
+ head_size=HEAD_SIZE,
137
+ block_size=BLOCK_SIZE,
138
+ num_blocks=NUM_BLOCKS,
139
+ dtype=DTYPE,
140
+ )
141
+
142
+ run_benchmark(
143
+ query_lens=[64] * 1,
144
+ context_lens=[64] * 1,
145
+ )
146
+ run_benchmark(
147
+ query_lens=[128] * 1,
148
+ context_lens=[128] * 1,
149
+ )
150
+ run_benchmark(
151
+ query_lens=[64] * 8,
152
+ context_lens=[64] * 8,
153
+ )
154
+ run_benchmark(
155
+ query_lens=[128] * 8,
156
+ context_lens=[128] * 8,
157
+ )
158
+ run_benchmark(
159
+ query_lens=[64, 32, 16],
160
+ context_lens=[128, 256, 64],
161
+ )
162
+ run_benchmark(
163
+ query_lens=[1024],
164
+ context_lens=[1024],
165
+ )
csrc/attention_kernels.cu
+11
-5
@@ -271,7 +271,8 @@ __device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
271
const float scale,
272
const int* __restrict__ block_table, // [num_seqs, max_num_blocks_per_seq]
273
const int context_len,
274
- const int max_num_blocks_per_seq) {
274
+ const int max_num_blocks_per_seq,
275
+ const int q_stride) {
276
constexpr int THREAD_GROUP_SIZE = WARP_SIZE / BLOCK_SIZE;
277
constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
278
const int thread_idx = threadIdx.x;
@@ -302,7 +303,8 @@ __device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
303
// For example, if the the thread group size is 4, then the first thread in the group
304
// has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
305
// th vectors of the query, and so on.
305
- const scalar_t* q_ptr = q + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
306
+ // NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
307
+ const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
308
Q_vec q_vecs[NUM_VECS_PER_THREAD];
309
#pragma unroll
310
for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
@@ -514,7 +516,8 @@ __global__ void multi_query_cached_kv_attention_kernel(
516
const float scale,
517
const int* __restrict__ block_tables, // [num_prompts, max_num_blocks_per_seq]
518
const int* __restrict__ context_lens, // [num_prompts]
517
- const int max_num_blocks_per_seq) {
519
+ const int max_num_blocks_per_seq,
520
+ const int q_stride) {
521
const int seq_idx = blockIdx.y;
522
const int prompt_idx = seq_prompt_mapping[seq_idx];
523
const int seq_start_idx = cu_query_lens[prompt_idx];
@@ -532,7 +535,8 @@ __global__ void multi_query_cached_kv_attention_kernel(
535
scale,
536
block_table,
537
context_len,
535
- max_num_blocks_per_seq);
538
+ max_num_blocks_per_seq,
539
+ q_stride);
540
}
541
542
} // namespace cacheflow
@@ -696,7 +700,8 @@ void single_query_cached_kv_attention(
700
scale, \
701
block_tables_ptr, \
702
context_lens_ptr, \
699
- max_num_blocks_per_seq);
703
+ max_num_blocks_per_seq, \
704
+ query_stride);
705
706
707
// TODO(woosuk): Tune NUM_THREADS.
@@ -719,6 +724,7 @@ void multi_query_cached_kv_attention_launcher(
724
int num_heads = query.size(1);
725
int head_size = query.size(2);
726
int max_num_blocks_per_seq = block_tables.size(1);
727
+ int query_stride = query.stride(0);
728
729
int* cu_query_lens_ptr = cu_query_lens.data_ptr<int>();
730
int* seq_prompt_mapping_ptr = seq_prompt_mapping.data_ptr<int>();
tests/kernels/attention.py
+5
-3
@@ -285,8 +285,9 @@ def test_multi_query_cached_kv_attention(
285
cu_query_lens.append(cu_query_lens[-1] + query_len)
286
num_total_tokens = cu_query_lens[-1]
287
288
- query = torch.randn(
289
- num_total_tokens, num_heads, head_size, dtype=dtype, device='cuda')
288
+ qkv = torch.randn(
289
+ num_total_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
290
+ query, _, _ = qkv.unbind(dim=1)
291
x = 16 // torch.tensor([], dtype=dtype).element_size()
292
key_block_shape = (num_heads, head_size // x, block_size, x)
293
key_cache = torch.randn(
@@ -314,7 +315,8 @@ def test_multi_query_cached_kv_attention(
315
block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
316
317
scale = float(1.0 / (head_size ** 0.5))
317
- output = torch.empty_like(query)
318
+ output = torch.empty(
319
+ num_total_tokens, num_heads, head_size, dtype=dtype, device='cuda')
320
321
attention_ops.multi_query_cached_kv_attention(
322
cu_query_lens,