@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c267b1a02c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add query stride to multi_query_cached_kv_attention & Add kernel benchmark script (#27)

* Add query stride to multi_query_cached_kv_attention * Add kernel benchmark script

Woosuk Kwon committed Apr 8, 2023 at 13:36 UTC c267b1a02c952b68a897c96201f32ad57e0b955e
3 files changed +181 -8
benchmark/benchmark_attention.py new
+165
@@ -0,0 +1,165 @@
1 +import functools
2 +import random
3 +import time
4 +from typing import List
5 +
6 +from flash_attn.flash_attn_interface import _flash_attn_forward
7 +import torch
8 +
9 +from cacheflow import attention_ops
10 +
11 +
12 +def benchmark(name, f, num_warmup = 10, num_iters = 100):
13 + for _ in range(num_warmup):
14 + f()
15 + torch.cuda.synchronize()
16 +
17 + start = time.time()
18 + for _ in range(num_iters):
19 + f()
20 + torch.cuda.synchronize()
21 + end = time.time()
22 + print(f'{name}: {(end - start) / num_iters * 1000:.3f} ms')
23 +
24 +
25 +@torch.inference_mode()
26 +def benchmark_multi_query_cached_kv_attention(
27 + query_lens: List[int],
28 + context_lens: List[int],
29 + num_heads: int,
30 + head_size: int,
31 + block_size: int,
32 + num_blocks: int,
33 + dtype: torch.dtype,
34 +) -> None:
35 + print(f'query_lens: {query_lens}, context_lens: {context_lens}, '
36 + f'num_heads: {num_heads}, head_size: {head_size}, block_size: '
37 + f'{block_size}, num_blocks: {num_blocks}, dtype: {dtype}')
38 + # Create query tensor.
39 + num_queries = len(query_lens)
40 + cu_query_lens = [0]
41 + for query_len in query_lens:
42 + cu_query_lens.append(cu_query_lens[-1] + query_len)
43 + num_total_tokens = cu_query_lens[-1]
44 + qkv = torch.randn(
45 + num_total_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
46 + query, _, _ = qkv.unbind(dim=1)
47 +
48 + # Create key and value cache.
49 + x = 16 // torch.tensor([], dtype=dtype).element_size()
50 + key_block_shape = (num_heads, head_size // x, block_size, x)
51 + key_cache = torch.randn(
52 + size=(num_blocks, *key_block_shape), dtype=dtype, device='cuda')
53 + value_block_shape = (num_heads, head_size, block_size)
54 + value_cache = torch.randn(
55 + size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
56 +
57 + # Create block tables.
58 + max_context_len = max(context_lens)
59 + max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
60 + block_tables = []
61 + for _ in range(num_queries):
62 + block_table = [
63 + random.randint(0, num_blocks - 1)
64 + for _ in range(max_num_blocks_per_seq)
65 + ]
66 + block_tables.append(block_table)
67 + block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
68 +
69 + # Create input and output data structures.
70 + cu_query_lens = torch.tensor(cu_query_lens, dtype=torch.int, device='cuda')
71 + context_len_tensor = torch.tensor(context_lens, dtype=torch.int, device='cuda')
72 + scale = float(1.0 / (head_size ** 0.5))
73 + output = torch.empty(
74 + num_total_tokens, num_heads, head_size, dtype=dtype, device='cuda')
75 +
76 + # Run our implementation.
77 + def run_ours():
78 + attention_ops.multi_query_cached_kv_attention(
79 + cu_query_lens,
80 + output,
81 + query,
82 + key_cache,
83 + value_cache,
84 + scale,
85 + block_tables,
86 + context_len_tensor,
87 + block_size,
88 + max_context_len,
89 + )
90 + benchmark('Ours', run_ours)
91 +
92 + # Upper bound: Flash attention.
93 + # Becuase Flash attention cannot read our own cache,
94 + # we make key and value tensors contiguous.
95 + num_kv_tokens = sum(context_lens)
96 + cu_context_lens = [0]
97 + for context_len in context_lens:
98 + cu_context_lens.append(cu_context_lens[-1] + context_len)
99 + cu_context_lens = torch.tensor(cu_context_lens, dtype=torch.int, device='cuda')
100 + qkv = torch.randn(
101 + num_kv_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
102 + _, key, value = qkv.unbind(dim=1)
103 + ref_output = torch.empty_like(output)
104 +
105 + # Run Flash attention.
106 + def run_flash_attn():
107 + _flash_attn_forward(
108 + query,
109 + key,
110 + value,
111 + ref_output,
112 + cu_query_lens,
113 + cu_context_lens,
114 + max(query_lens),
115 + max_context_len,
116 + dropout_p=0.0,
117 + softmax_scale=scale,
118 + causal=True,
119 + return_softmax=False,
120 + )
121 + benchmark('Flash attention', run_flash_attn)
122 +
123 +
124 +if __name__ == '__main__':
125 + BLOCK_SIZE = 8
126 + NUM_BLOCKS = 1024
127 + DTYPE = torch.half
128 +
129 + # LLaMA-13B and OPT-13B
130 + NUM_HEADS = 40
131 + HEAD_SIZE = 128
132 +
133 + run_benchmark = functools.partial(
134 + benchmark_multi_query_cached_kv_attention,
135 + num_heads=NUM_HEADS,
136 + head_size=HEAD_SIZE,
137 + block_size=BLOCK_SIZE,
138 + num_blocks=NUM_BLOCKS,
139 + dtype=DTYPE,
140 + )
141 +
142 + run_benchmark(
143 + query_lens=[64] * 1,
144 + context_lens=[64] * 1,
145 + )
146 + run_benchmark(
147 + query_lens=[128] * 1,
148 + context_lens=[128] * 1,
149 + )
150 + run_benchmark(
151 + query_lens=[64] * 8,
152 + context_lens=[64] * 8,
153 + )
154 + run_benchmark(
155 + query_lens=[128] * 8,
156 + context_lens=[128] * 8,
157 + )
158 + run_benchmark(
159 + query_lens=[64, 32, 16],
160 + context_lens=[128, 256, 64],
161 + )
162 + run_benchmark(
163 + query_lens=[1024],
164 + context_lens=[1024],
165 + )
csrc/attention_kernels.cu
+11 -5
@@ -271,7 +271,8 @@ __device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
271 const float scale,
272 const int* __restrict__ block_table, // [num_seqs, max_num_blocks_per_seq]
273 const int context_len,
274 - const int max_num_blocks_per_seq) {
274 + const int max_num_blocks_per_seq,
275 + const int q_stride) {
276 constexpr int THREAD_GROUP_SIZE = WARP_SIZE / BLOCK_SIZE;
277 constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
278 const int thread_idx = threadIdx.x;
@@ -302,7 +303,8 @@ __device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
303 // For example, if the the thread group size is 4, then the first thread in the group
304 // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
305 // th vectors of the query, and so on.
305 - const scalar_t* q_ptr = q + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
306 + // NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
307 + const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
308 Q_vec q_vecs[NUM_VECS_PER_THREAD];
309 #pragma unroll
310 for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
@@ -514,7 +516,8 @@ __global__ void multi_query_cached_kv_attention_kernel(
516 const float scale,
517 const int* __restrict__ block_tables, // [num_prompts, max_num_blocks_per_seq]
518 const int* __restrict__ context_lens, // [num_prompts]
517 - const int max_num_blocks_per_seq) {
519 + const int max_num_blocks_per_seq,
520 + const int q_stride) {
521 const int seq_idx = blockIdx.y;
522 const int prompt_idx = seq_prompt_mapping[seq_idx];
523 const int seq_start_idx = cu_query_lens[prompt_idx];
@@ -532,7 +535,8 @@ __global__ void multi_query_cached_kv_attention_kernel(
535 scale,
536 block_table,
537 context_len,
535 - max_num_blocks_per_seq);
538 + max_num_blocks_per_seq,
539 + q_stride);
540 }
541
542 } // namespace cacheflow
@@ -696,7 +700,8 @@ void single_query_cached_kv_attention(
700 scale, \
701 block_tables_ptr, \
702 context_lens_ptr, \
699 - max_num_blocks_per_seq);
703 + max_num_blocks_per_seq, \
704 + query_stride);
705
706
707 // TODO(woosuk): Tune NUM_THREADS.
@@ -719,6 +724,7 @@ void multi_query_cached_kv_attention_launcher(
724 int num_heads = query.size(1);
725 int head_size = query.size(2);
726 int max_num_blocks_per_seq = block_tables.size(1);
727 + int query_stride = query.stride(0);
728
729 int* cu_query_lens_ptr = cu_query_lens.data_ptr<int>();
730 int* seq_prompt_mapping_ptr = seq_prompt_mapping.data_ptr<int>();
tests/kernels/attention.py
+5 -3
@@ -285,8 +285,9 @@ def test_multi_query_cached_kv_attention(
285 cu_query_lens.append(cu_query_lens[-1] + query_len)
286 num_total_tokens = cu_query_lens[-1]
287
288 - query = torch.randn(
289 - num_total_tokens, num_heads, head_size, dtype=dtype, device='cuda')
288 + qkv = torch.randn(
289 + num_total_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
290 + query, _, _ = qkv.unbind(dim=1)
291 x = 16 // torch.tensor([], dtype=dtype).element_size()
292 key_block_shape = (num_heads, head_size // x, block_size, x)
293 key_cache = torch.randn(
@@ -314,7 +315,8 @@ def test_multi_query_cached_kv_attention(
315 block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
316
317 scale = float(1.0 / (head_size ** 0.5))
317 - output = torch.empty_like(query)
318 + output = torch.empty(
319 + num_total_tokens, num_heads, head_size, dtype=dtype, device='cuda')
320
321 attention_ops.multi_query_cached_kv_attention(
322 cu_query_lens,