A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Implement PagedAttention V2 (#1348)
Woosuk Kwon committed
Oct 16, 2023 at 00:59 UTC
928de46888b9b257dfa491047a7d9cd199ca585b
6 files changed
+764
-139
benchmarks/kernels/benchmark_paged_attention.py
new
+197
@@ -0,0 +1,197 @@
1
+import argparse
2
+import random
3
+import time
4
+
5
+import torch
6
+
7
+from vllm import attention_ops
8
+
9
+NUM_BLOCKS = 1024
10
+PARTITION_SIZE = 512
11
+
12
+
13
+@torch.inference_mode()
14
+def main(
15
+ version: str,
16
+ num_seqs: int,
17
+ context_len: int,
18
+ num_query_heads: int,
19
+ num_kv_heads: int,
20
+ head_size: int,
21
+ use_alibi: bool,
22
+ block_size: int,
23
+ dtype: torch.dtype,
24
+ seed: int,
25
+ do_profile: bool,
26
+) -> None:
27
+ random.seed(seed)
28
+ torch.random.manual_seed(seed)
29
+ torch.cuda.manual_seed(seed)
30
+
31
+ scale = float(1.0 / (head_size**0.5))
32
+ query = torch.empty(num_seqs,
33
+ num_query_heads,
34
+ head_size,
35
+ dtype=dtype,
36
+ device="cuda")
37
+ query.uniform_(-scale, scale)
38
+
39
+ assert num_query_heads % num_kv_heads == 0
40
+ num_queries_per_kv = num_query_heads // num_kv_heads
41
+ head_mapping = torch.repeat_interleave(
42
+ torch.arange(num_kv_heads, dtype=torch.int32, device="cuda"),
43
+ num_queries_per_kv)
44
+ alibi_slopes = None
45
+ if use_alibi:
46
+ alibi_slopes = torch.randn(num_query_heads,
47
+ dtype=torch.float,
48
+ device="cuda")
49
+
50
+ context_lens = [context_len for _ in range(num_seqs)]
51
+ max_context_len = max(context_lens)
52
+ context_lens = torch.tensor(context_lens, dtype=torch.int, device="cuda")
53
+
54
+ # Create the block tables.
55
+ max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
56
+ block_tables = []
57
+ for _ in range(num_seqs):
58
+ block_table = [
59
+ random.randint(0, NUM_BLOCKS - 1)
60
+ for _ in range(max_num_blocks_per_seq)
61
+ ]
62
+ block_tables.append(block_table)
63
+ block_tables = torch.tensor(block_tables, dtype=torch.int, device="cuda")
64
+
65
+ # Create the KV cache.
66
+ x = 16 // torch.tensor([], dtype=dtype).element_size()
67
+ key_cache_shape = (NUM_BLOCKS, num_kv_heads, head_size // x, block_size, x)
68
+ key_cache = torch.empty(size=key_cache_shape, dtype=dtype, device="cuda")
69
+ key_cache.uniform_(-scale, scale)
70
+ value_cache_shape = (NUM_BLOCKS, num_kv_heads, head_size, block_size)
71
+ value_cache = torch.empty(size=value_cache_shape,
72
+ dtype=dtype,
73
+ device="cuda")
74
+ value_cache.uniform_(-scale, scale)
75
+
76
+ # Prepare for the paged attention kernel.
77
+ output = torch.empty_like(query)
78
+ if version == "v2":
79
+ num_partitions = ((max_context_len + PARTITION_SIZE - 1) //
80
+ PARTITION_SIZE)
81
+ tmp_output = torch.empty(
82
+ size=(num_seqs, num_query_heads, num_partitions, head_size),
83
+ dtype=output.dtype,
84
+ device=output.device,
85
+ )
86
+ exp_sums = torch.empty(
87
+ size=(num_seqs, num_query_heads, num_partitions),
88
+ dtype=torch.float32,
89
+ device=output.device,
90
+ )
91
+ max_logits = torch.empty_like(exp_sums)
92
+
93
+ def run_benchmark(num_iters: int, profile: bool = False) -> float:
94
+ torch.cuda.synchronize()
95
+ if profile:
96
+ torch.cuda.cudart().cudaProfilerStart()
97
+ start_time = time.perf_counter()
98
+
99
+ for _ in range(num_iters):
100
+ if version == "v1":
101
+ attention_ops.paged_attention_v1(
102
+ output,
103
+ query,
104
+ key_cache,
105
+ value_cache,
106
+ head_mapping,
107
+ scale,
108
+ block_tables,
109
+ context_lens,
110
+ block_size,
111
+ max_context_len,
112
+ alibi_slopes,
113
+ )
114
+ elif version == "v2":
115
+ attention_ops.paged_attention_v2(
116
+ output,
117
+ exp_sums,
118
+ max_logits,
119
+ tmp_output,
120
+ query,
121
+ key_cache,
122
+ value_cache,
123
+ head_mapping,
124
+ scale,
125
+ block_tables,
126
+ context_lens,
127
+ block_size,
128
+ max_context_len,
129
+ alibi_slopes,
130
+ )
131
+ else:
132
+ raise ValueError(f"Invalid version: {version}")
133
+ torch.cuda.synchronize()
134
+
135
+ end_time = time.perf_counter()
136
+ if profile:
137
+ torch.cuda.cudart().cudaProfilerStart()
138
+ return (end_time - start_time) / num_iters
139
+
140
+ # Warmup.
141
+ print("Warming up...")
142
+ run_benchmark(num_iters=3, profile=False)
143
+
144
+ # Benchmark.
145
+ if do_profile:
146
+ latency = run_benchmark(num_iters=1, profile=True)
147
+ else:
148
+ latency = run_benchmark(num_iters=100, profile=False)
149
+ print(f"Kernel running time: {latency * 1000000:.3f} us")
150
+
151
+
152
+if __name__ == '__main__':
153
+ parser = argparse.ArgumentParser(
154
+ description="Benchmark the paged attention kernel.")
155
+ parser.add_argument("--version",
156
+ type=str,
157
+ choices=["v1", "v2"],
158
+ default="v2")
159
+ parser.add_argument("--batch-size", type=int, default=8)
160
+ parser.add_argument("--context-len", type=int, default=4096)
161
+ parser.add_argument("--num-query-heads", type=int, default=64)
162
+ parser.add_argument("--num-kv-heads", type=int, default=8)
163
+ parser.add_argument("--head-size",
164
+ type=int,
165
+ choices=[64, 80, 96, 112, 128, 256],
166
+ default=128)
167
+ parser.add_argument("--block-size", type=int, choices=[16, 32], default=16)
168
+ parser.add_argument("--use-alibi", action="store_true")
169
+ parser.add_argument("--dtype",
170
+ type=str,
171
+ choices=["half", "bfloat16", "float"],
172
+ default="half")
173
+ parser.add_argument("--seed", type=int, default=0)
174
+ parser.add_argument("--profile", action="store_true")
175
+ args = parser.parse_args()
176
+ print(args)
177
+
178
+ if args.num_query_heads % args.num_kv_heads != 0:
179
+ raise ValueError("num_query_heads must be divisible by num_kv_heads")
180
+ dtype_to_torch_dtype = {
181
+ "half": torch.half,
182
+ "bfloat16": torch.bfloat16,
183
+ "float": torch.float,
184
+ }
185
+ main(
186
+ version=args.version,
187
+ num_seqs=args.batch_size,
188
+ context_len=args.context_len,
189
+ num_query_heads=args.num_query_heads,
190
+ num_kv_heads=args.num_kv_heads,
191
+ head_size=args.head_size,
192
+ block_size=args.block_size,
193
+ use_alibi=args.use_alibi,
194
+ dtype=dtype_to_torch_dtype[args.dtype],
195
+ seed=args.seed,
196
+ do_profile=args.profile,
197
+ )
csrc/attention.cpp
+24
-4
@@ -1,7 +1,7 @@
1
#include <torch/extension.h>
2
#include <c10/util/Optional.h>
3
4
-void single_query_cached_kv_attention(
4
+void paged_attention_v1(
5
torch::Tensor& out,
6
torch::Tensor& query,
7
torch::Tensor& key_cache,
@@ -14,9 +14,29 @@ void single_query_cached_kv_attention(
14
int max_context_len,
15
const c10::optional<torch::Tensor>& alibi_slopes);
16
17
+void paged_attention_v2(
18
+ torch::Tensor& out,
19
+ torch::Tensor& exp_sums,
20
+ torch::Tensor& max_logits,
21
+ torch::Tensor& tmp_out,
22
+ torch::Tensor& query,
23
+ torch::Tensor& key_cache,
24
+ torch::Tensor& value_cache,
25
+ torch::Tensor& head_mapping,
26
+ float scale,
27
+ torch::Tensor& block_tables,
28
+ torch::Tensor& context_lens,
29
+ int block_size,
30
+ int max_context_len,
31
+ const c10::optional<torch::Tensor>& alibi_slopes);
32
+
33
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
34
m.def(
19
- "single_query_cached_kv_attention",
20
- &single_query_cached_kv_attention,
21
- "Compute the attention between an input query and the cached key/value tensors");
35
+ "paged_attention_v1",
36
+ &paged_attention_v1,
37
+ "Compute the attention between an input query and the cached keys/values using PagedAttention.");
38
+ m.def(
39
+ "paged_attention_v2",
40
+ &paged_attention_v2,
41
+ "PagedAttention V2.");
42
}
csrc/attention/attention_kernels.cu
+413
-71
@@ -26,6 +26,7 @@
26
#define WARP_SIZE 32
27
#define MAX(a, b) ((a) > (b) ? (a) : (b))
28
#define MIN(a, b) ((a) < (b) ? (a) : (b))
29
+#define DIVIDE_ROUND_UP(a, b) (((a) + (b) - 1) / (b))
30
31
namespace vllm {
32
@@ -65,14 +66,18 @@ inline __device__ float block_sum(float* red_smem, float sum) {
66
return __shfl_sync(uint32_t(-1), sum, 0);
67
}
68
68
-// Grid: (num_heads, num_seqs).
69
+// TODO(woosuk): Merge the last two dimensions of the grid.
70
+// Grid: (num_heads, num_seqs, max_num_partitions).
71
template<
72
typename scalar_t,
73
int HEAD_SIZE,
74
int BLOCK_SIZE,
73
- int NUM_THREADS>
74
-__global__ void single_query_cached_kv_attention_kernel(
75
- scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
75
+ int NUM_THREADS,
76
+ int PARTITION_SIZE = 0> // Zero means no partitioning.
77
+__device__ void paged_attention_kernel(
78
+ float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
79
+ float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
80
+ scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions, head_size]
81
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
82
const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
83
const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
@@ -85,10 +90,33 @@ __global__ void single_query_cached_kv_attention_kernel(
90
const int q_stride,
91
const int kv_block_stride,
92
const int kv_head_stride) {
93
+ const int seq_idx = blockIdx.y;
94
+ const int partition_idx = blockIdx.z;
95
+ const int max_num_partitions = gridDim.z;
96
+ constexpr bool USE_PARTITIONING = PARTITION_SIZE > 0;
97
+ const int context_len = context_lens[seq_idx];
98
+ if (USE_PARTITIONING && partition_idx * PARTITION_SIZE >= context_len) {
99
+ // No work to do. Terminate the thread block.
100
+ return;
101
+ }
102
+
103
+ const int num_context_blocks = DIVIDE_ROUND_UP(context_len, BLOCK_SIZE);
104
+ const int num_blocks_per_partition = USE_PARTITIONING ? PARTITION_SIZE / BLOCK_SIZE : num_context_blocks;
105
+
106
+ // [start_block_idx, end_block_idx) is the range of blocks to process.
107
+ const int start_block_idx = USE_PARTITIONING ? partition_idx * num_blocks_per_partition : 0;
108
+ const int end_block_idx = MIN(start_block_idx + num_blocks_per_partition, num_context_blocks);
109
+ const int num_blocks = end_block_idx - start_block_idx;
110
+
111
+ // [start_token_idx, end_token_idx) is the range of tokens to process.
112
+ const int start_token_idx = start_block_idx * BLOCK_SIZE;
113
+ const int end_token_idx = MIN(start_token_idx + num_blocks * BLOCK_SIZE, context_len);
114
+ const int num_tokens = end_token_idx - start_token_idx;
115
+
116
constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
117
constexpr int NUM_THREAD_GROUPS = NUM_THREADS / THREAD_GROUP_SIZE; // Note: This assumes THREAD_GROUP_SIZE divides NUM_THREADS
118
assert(NUM_THREADS % THREAD_GROUP_SIZE == 0);
91
- constexpr int NUM_TOKENS_PER_THREAD_GROUP = (BLOCK_SIZE + WARP_SIZE - 1) / WARP_SIZE;
119
+ constexpr int NUM_TOKENS_PER_THREAD_GROUP = DIVIDE_ROUND_UP(BLOCK_SIZE, WARP_SIZE);
120
constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
121
const int thread_idx = threadIdx.x;
122
const int warp_idx = thread_idx / WARP_SIZE;
@@ -97,7 +125,6 @@ __global__ void single_query_cached_kv_attention_kernel(
125
const int head_idx = blockIdx.x;
126
const int num_heads = gridDim.x;
127
const int kv_head_idx = head_mapping[head_idx];
100
- const int seq_idx = blockIdx.y;
128
const float alibi_slope = alibi_slopes == nullptr ? 0.f : alibi_slopes[head_idx];
129
130
// A vector type to store a part of a key or a query.
@@ -142,15 +169,12 @@ __global__ void single_query_cached_kv_attention_kernel(
169
constexpr int x = 16 / sizeof(scalar_t);
170
float qk_max = -FLT_MAX;
171
145
- const int* block_table = block_tables + seq_idx * max_num_blocks_per_seq;
146
- const int context_len = context_lens[seq_idx];
147
- const int num_blocks = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
148
-
172
// Iterate over the key blocks.
173
// Each warp fetches a block of keys for each iteration.
174
// Each thread group in a warp fetches a key from the block, and computes
175
// dot product with the query.
153
- for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
176
+ const int* block_table = block_tables + seq_idx * max_num_blocks_per_seq;
177
+ for (int block_idx = start_block_idx + warp_idx; block_idx < end_block_idx; block_idx += NUM_WARPS) {
178
const int physical_block_number = block_table[block_idx];
179
180
// Load a key to registers.
@@ -184,7 +208,7 @@ __global__ void single_query_cached_kv_attention_kernel(
208
// Store the partial reductions to shared memory.
209
// NOTE(woosuk): It is required to zero out the masked logits.
210
const bool mask = token_idx >= context_len;
187
- logits[token_idx] = mask ? 0.f : qk;
211
+ logits[token_idx - start_token_idx] = mask ? 0.f : qk;
212
// Update the max value.
213
qk_max = mask ? qk_max : fmaxf(qk_max, qk);
214
}
@@ -215,7 +239,7 @@ __global__ void single_query_cached_kv_attention_kernel(
239
240
// Get the sum of the exp values.
241
float exp_sum = 0.f;
218
- for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
242
+ for (int i = thread_idx; i < num_tokens; i += NUM_THREADS) {
243
float val = __expf(logits[i] - qk_max);
244
logits[i] = val;
245
exp_sum += val;
@@ -224,11 +248,23 @@ __global__ void single_query_cached_kv_attention_kernel(
248
249
// Compute softmax.
250
const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
227
- for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
251
+ for (int i = thread_idx; i < num_tokens; i += NUM_THREADS) {
252
logits[i] *= inv_sum;
253
}
254
__syncthreads();
255
256
+ // If partitioning is enabled, store the max logit and exp_sum.
257
+ if (USE_PARTITIONING && thread_idx == 0) {
258
+ float* max_logits_ptr = max_logits + seq_idx * num_heads * max_num_partitions
259
+ + head_idx * max_num_partitions
260
+ + partition_idx;
261
+ *max_logits_ptr = qk_max;
262
+ float* exp_sums_ptr = exp_sums + seq_idx * num_heads * max_num_partitions
263
+ + head_idx * max_num_partitions
264
+ + partition_idx;
265
+ *exp_sums_ptr = exp_sum;
266
+ }
267
+
268
// Each thread will fetch 16 bytes from the value cache at a time.
269
constexpr int V_VEC_SIZE = MIN(16 / sizeof(scalar_t), BLOCK_SIZE);
270
using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
@@ -237,7 +273,7 @@ __global__ void single_query_cached_kv_attention_kernel(
273
274
constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
275
constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
240
- constexpr int NUM_ROWS_PER_THREAD = (HEAD_SIZE + NUM_ROWS_PER_ITER - 1) / NUM_ROWS_PER_ITER;
276
+ constexpr int NUM_ROWS_PER_THREAD = DIVIDE_ROUND_UP(HEAD_SIZE, NUM_ROWS_PER_ITER);
277
278
// NOTE(woosuk): We use FP32 for the accumulator for better accuracy.
279
float accs[NUM_ROWS_PER_THREAD];
@@ -248,12 +284,12 @@ __global__ void single_query_cached_kv_attention_kernel(
284
285
scalar_t zero_value;
286
zero(zero_value);
251
- for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
287
+ for (int block_idx = start_block_idx + warp_idx; block_idx < end_block_idx; block_idx += NUM_WARPS) {
288
const int physical_block_number = block_table[block_idx];
289
const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
290
const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
291
L_vec logits_vec;
256
- from_float(logits_vec, *reinterpret_cast<Float_L_vec*>(logits + token_idx));
292
+ from_float(logits_vec, *reinterpret_cast<Float_L_vec*>(logits + token_idx - start_token_idx));
293
294
const scalar_t* v_ptr = v_cache + physical_block_number * kv_block_stride
295
+ kv_head_idx * kv_head_stride;
@@ -263,7 +299,7 @@ __global__ void single_query_cached_kv_attention_kernel(
299
if (row_idx < HEAD_SIZE) {
300
const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
301
V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
266
- if (block_idx == num_blocks - 1) {
302
+ if (block_idx == num_context_blocks - 1) {
303
// NOTE(woosuk): When v_vec contains the tokens that are out of the context,
304
// we should explicitly zero out the values since they may contain NaNs.
305
// See https://github.com/vllm-project/vllm/issues/641#issuecomment-1682544472
@@ -327,7 +363,9 @@ __global__ void single_query_cached_kv_attention_kernel(
363
364
// Write the final output.
365
if (warp_idx == 0) {
330
- scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
366
+ scalar_t* out_ptr = out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE
367
+ + head_idx * max_num_partitions * HEAD_SIZE
368
+ + partition_idx * HEAD_SIZE;
369
#pragma unroll
370
for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
371
const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
@@ -338,13 +376,167 @@ __global__ void single_query_cached_kv_attention_kernel(
376
}
377
}
378
379
+// Grid: (num_heads, num_seqs, 1).
380
+template<
381
+ typename scalar_t,
382
+ int HEAD_SIZE,
383
+ int BLOCK_SIZE,
384
+ int NUM_THREADS>
385
+__global__ void paged_attention_v1_kernel(
386
+ scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
387
+ const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
388
+ const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
389
+ const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
390
+ const int* __restrict__ head_mapping, // [num_heads]
391
+ const float scale,
392
+ const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
393
+ const int* __restrict__ context_lens, // [num_seqs]
394
+ const int max_num_blocks_per_seq,
395
+ const float* __restrict__ alibi_slopes, // [num_heads]
396
+ const int q_stride,
397
+ const int kv_block_stride,
398
+ const int kv_head_stride) {
399
+ paged_attention_kernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>(
400
+ /* exp_sums */ nullptr, /* max_logits */ nullptr,
401
+ out, q, k_cache, v_cache, head_mapping, scale, block_tables, context_lens,
402
+ max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride, kv_head_stride);
403
+}
404
+
405
+// Grid: (num_heads, num_seqs, max_num_partitions).
406
+template<
407
+ typename scalar_t,
408
+ int HEAD_SIZE,
409
+ int BLOCK_SIZE,
410
+ int NUM_THREADS,
411
+ int PARTITION_SIZE>
412
+__global__ void paged_attention_v2_kernel(
413
+ float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
414
+ float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
415
+ scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
416
+ const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
417
+ const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
418
+ const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
419
+ const int* __restrict__ head_mapping, // [num_heads]
420
+ const float scale,
421
+ const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
422
+ const int* __restrict__ context_lens, // [num_seqs]
423
+ const int max_num_blocks_per_seq,
424
+ const float* __restrict__ alibi_slopes, // [num_heads]
425
+ const int q_stride,
426
+ const int kv_block_stride,
427
+ const int kv_head_stride) {
428
+ paged_attention_kernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, PARTITION_SIZE>(
429
+ exp_sums, max_logits, tmp_out, q, k_cache, v_cache, head_mapping, scale,
430
+ block_tables, context_lens, max_num_blocks_per_seq, alibi_slopes,
431
+ q_stride, kv_block_stride, kv_head_stride);
432
+}
433
+
434
+// Grid: (num_heads, num_seqs).
435
+template<
436
+ typename scalar_t,
437
+ int HEAD_SIZE,
438
+ int NUM_THREADS,
439
+ int PARTITION_SIZE>
440
+__global__ void paged_attention_v2_reduce_kernel(
441
+ scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
442
+ const float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
443
+ const float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
444
+ const scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
445
+ const int* __restrict__ context_lens, // [num_seqs]
446
+ const int max_num_partitions) {
447
+ const int num_heads = gridDim.x;
448
+ const int head_idx = blockIdx.x;
449
+ const int seq_idx = blockIdx.y;
450
+ const int context_len = context_lens[seq_idx];
451
+ const int num_partitions = DIVIDE_ROUND_UP(context_len, PARTITION_SIZE);
452
+ if (num_partitions == 1) {
453
+ // No need to reduce. Only copy tmp_out to out.
454
+ scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
455
+ const scalar_t* tmp_out_ptr = tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE
456
+ + head_idx * max_num_partitions * HEAD_SIZE;
457
+ for (int i = threadIdx.x; i < HEAD_SIZE; i += blockDim.x) {
458
+ out_ptr[i] = tmp_out_ptr[i];
459
+ }
460
+ // Terminate the thread block.
461
+ return;
462
+ }
463
+
464
+ constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
465
+ const int warp_idx = threadIdx.x / WARP_SIZE;
466
+ const int lane = threadIdx.x % WARP_SIZE;
467
+
468
+ // Size: 2 * num_partitions.
469
+ extern __shared__ char shared_mem[];
470
+ // Workspace for reduction.
471
+ __shared__ float red_smem[2 * NUM_WARPS];
472
+
473
+ // Load max logits to shared memory.
474
+ float* shared_max_logits = reinterpret_cast<float*>(shared_mem);
475
+ const float* max_logits_ptr = max_logits + seq_idx * num_heads * max_num_partitions
476
+ + head_idx * max_num_partitions;
477
+ float max_logit = -FLT_MAX;
478
+ for (int i = threadIdx.x; i < num_partitions; i += blockDim.x) {
479
+ const float l = max_logits_ptr[i];
480
+ shared_max_logits[i] = l;
481
+ max_logit = fmaxf(max_logit, l);
482
+ }
483
+ __syncthreads();
484
+
485
+ // Get the global max logit.
486
+ // Reduce within the warp.
487
+#pragma unroll
488
+ for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
489
+ max_logit = fmaxf(max_logit, __shfl_xor_sync(uint32_t(-1), max_logit, mask));
490
+ }
491
+ if (lane == 0) {
492
+ red_smem[warp_idx] = max_logit;
493
+ }
494
+ __syncthreads();
495
+ // Reduce across warps.
496
+ max_logit = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
497
+#pragma unroll
498
+ for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
499
+ max_logit = fmaxf(max_logit, __shfl_xor_sync(uint32_t(-1), max_logit, mask));
500
+ }
501
+ // Broadcast the max value to all threads.
502
+ max_logit = __shfl_sync(uint32_t(-1), max_logit, 0);
503
+
504
+ // Load rescaled exp sums to shared memory.
505
+ float* shared_exp_sums = reinterpret_cast<float*>(shared_mem + sizeof(float) * num_partitions);
506
+ const float* exp_sums_ptr = exp_sums + seq_idx * num_heads * max_num_partitions
507
+ + head_idx * max_num_partitions;
508
+ float global_exp_sum = 0.0f;
509
+ for (int i = threadIdx.x; i < num_partitions; i += blockDim.x) {
510
+ float l = shared_max_logits[i];
511
+ float rescaled_exp_sum = exp_sums_ptr[i] * expf(l - max_logit);
512
+ global_exp_sum += rescaled_exp_sum;
513
+ shared_exp_sums[i] = rescaled_exp_sum;
514
+ }
515
+ __syncthreads();
516
+ global_exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], global_exp_sum);
517
+ const float inv_global_exp_sum = __fdividef(1.0f, global_exp_sum + 1e-6f);
518
+
519
+ // Aggregate tmp_out to out.
520
+ const scalar_t* tmp_out_ptr = tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE
521
+ + head_idx * max_num_partitions * HEAD_SIZE;
522
+ scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
523
+#pragma unroll
524
+ for (int i = threadIdx.x; i < HEAD_SIZE; i += NUM_THREADS) {
525
+ float acc = 0.0f;
526
+ for (int j = 0; j < num_partitions; ++j) {
527
+ acc += to_float(tmp_out_ptr[j * HEAD_SIZE + i]) * shared_exp_sums[j] * inv_global_exp_sum;
528
+ }
529
+ from_float(out_ptr[i], acc);
530
+ }
531
+}
532
+
533
} // namespace vllm
534
343
-#define LAUNCH_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
535
+#define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
536
cudaFuncSetAttribute( \
345
- vllm::single_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>, \
346
- cudaFuncAttributeMaxDynamicSharedMemorySize, shared_mem_size); \
347
- vllm::single_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
537
+ vllm::paged_attention_v1_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>, \
538
+ cudaFuncAttributeMaxDynamicSharedMemorySize, shared_mem_size); \
539
+ vllm::paged_attention_v1_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
540
<<<grid, block, shared_mem_size, stream>>>( \
541
out_ptr, \
542
query_ptr, \
@@ -365,7 +557,7 @@ template<
557
typename T,
558
int BLOCK_SIZE,
559
int NUM_THREADS = 128>
368
-void single_query_cached_kv_attention_launcher(
560
+void paged_attention_v1_launcher(
561
torch::Tensor& out,
562
torch::Tensor& query,
563
torch::Tensor& key_cache,
@@ -401,45 +593,206 @@ void single_query_cached_kv_attention_launcher(
593
int* context_lens_ptr = context_lens.data_ptr<int>();
594
595
constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
404
- int padded_max_context_len = ((max_context_len + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
596
+ int padded_max_context_len = DIVIDE_ROUND_UP(max_context_len, BLOCK_SIZE) * BLOCK_SIZE;
597
int logits_size = padded_max_context_len * sizeof(float);
598
int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
599
// Python-side check in vllm.worker.worker._check_if_can_support_max_seq_len
600
// Keep that in sync with the logic here!
601
int shared_mem_size = std::max(logits_size, outputs_size);
602
411
- dim3 grid(num_heads, num_seqs);
603
+ dim3 grid(num_heads, num_seqs, 1);
604
+ dim3 block(NUM_THREADS);
605
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
606
+ switch (head_size) {
607
+ // NOTE(woosuk): To reduce the compilation time, we only compile for the
608
+ // head sizes that we use in the model. However, we can easily extend this
609
+ // to support any head size which is a multiple of 16.
610
+ case 64:
611
+ LAUNCH_PAGED_ATTENTION_V1(64);
612
+ break;
613
+ case 80:
614
+ LAUNCH_PAGED_ATTENTION_V1(80);
615
+ break;
616
+ case 96:
617
+ LAUNCH_PAGED_ATTENTION_V1(96);
618
+ break;
619
+ case 112:
620
+ LAUNCH_PAGED_ATTENTION_V1(112);
621
+ break;
622
+ case 128:
623
+ LAUNCH_PAGED_ATTENTION_V1(128);
624
+ break;
625
+ case 256:
626
+ LAUNCH_PAGED_ATTENTION_V1(256);
627
+ break;
628
+ default:
629
+ TORCH_CHECK(false, "Unsupported head size: ", head_size);
630
+ break;
631
+ }
632
+}
633
+
634
+#define CALL_V1_LAUNCHER(T, BLOCK_SIZE) \
635
+ paged_attention_v1_launcher<T, BLOCK_SIZE>( \
636
+ out, \
637
+ query, \
638
+ key_cache, \
639
+ value_cache, \
640
+ head_mapping, \
641
+ scale, \
642
+ block_tables, \
643
+ context_lens, \
644
+ max_context_len, \
645
+ alibi_slopes);
646
+
647
+// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
648
+// 1, 2, 4, 64, 128, 256.
649
+#define CALL_V1_LAUNCHER_BLOCK_SIZE(T) \
650
+ switch (block_size) { \
651
+ case 8: \
652
+ CALL_V1_LAUNCHER(T, 8); \
653
+ break; \
654
+ case 16: \
655
+ CALL_V1_LAUNCHER(T, 16); \
656
+ break; \
657
+ case 32: \
658
+ CALL_V1_LAUNCHER(T, 32); \
659
+ break; \
660
+ default: \
661
+ TORCH_CHECK(false, "Unsupported block size: ", block_size); \
662
+ break; \
663
+ }
664
+
665
+void paged_attention_v1(
666
+ torch::Tensor& out, // [num_seqs, num_heads, head_size]
667
+ torch::Tensor& query, // [num_seqs, num_heads, head_size]
668
+ torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
669
+ torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
670
+ torch::Tensor& head_mapping, // [num_heads]
671
+ float scale,
672
+ torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
673
+ torch::Tensor& context_lens, // [num_seqs]
674
+ int block_size,
675
+ int max_context_len,
676
+ const c10::optional<torch::Tensor>& alibi_slopes) {
677
+ if (query.dtype() == at::ScalarType::Float) {
678
+ CALL_V1_LAUNCHER_BLOCK_SIZE(float);
679
+ } else if (query.dtype() == at::ScalarType::Half) {
680
+ CALL_V1_LAUNCHER_BLOCK_SIZE(uint16_t);
681
+ } else if (query.dtype() == at::ScalarType::BFloat16) {
682
+ CALL_V1_LAUNCHER_BLOCK_SIZE(__nv_bfloat16);
683
+ } else {
684
+ TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
685
+ }
686
+}
687
+
688
+#define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
689
+ vllm::paged_attention_v2_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, PARTITION_SIZE> \
690
+ <<<grid, block, shared_mem_size, stream>>>( \
691
+ exp_sums_ptr, \
692
+ max_logits_ptr, \
693
+ tmp_out_ptr, \
694
+ query_ptr, \
695
+ key_cache_ptr, \
696
+ value_cache_ptr, \
697
+ head_mapping_ptr, \
698
+ scale, \
699
+ block_tables_ptr, \
700
+ context_lens_ptr, \
701
+ max_num_blocks_per_seq, \
702
+ alibi_slopes_ptr, \
703
+ q_stride, \
704
+ kv_block_stride, \
705
+ kv_head_stride); \
706
+ vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, PARTITION_SIZE> \
707
+ <<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
708
+ out_ptr, \
709
+ exp_sums_ptr, \
710
+ max_logits_ptr, \
711
+ tmp_out_ptr, \
712
+ context_lens_ptr, \
713
+ max_num_partitions);
714
+
715
+template<
716
+ typename T,
717
+ int BLOCK_SIZE,
718
+ int NUM_THREADS = 128,
719
+ int PARTITION_SIZE = 512>
720
+void paged_attention_v2_launcher(
721
+ torch::Tensor& out,
722
+ torch::Tensor& exp_sums,
723
+ torch::Tensor& max_logits,
724
+ torch::Tensor& tmp_out,
725
+ torch::Tensor& query,
726
+ torch::Tensor& key_cache,
727
+ torch::Tensor& value_cache,
728
+ torch::Tensor& head_mapping,
729
+ float scale,
730
+ torch::Tensor& block_tables,
731
+ torch::Tensor& context_lens,
732
+ int max_context_len,
733
+ const c10::optional<torch::Tensor>& alibi_slopes) {
734
+ int num_seqs = query.size(0);
735
+ int num_heads = query.size(1);
736
+ int head_size = query.size(2);
737
+ int max_num_blocks_per_seq = block_tables.size(1);
738
+ int q_stride = query.stride(0);
739
+ int kv_block_stride = key_cache.stride(0);
740
+ int kv_head_stride = key_cache.stride(1);
741
+
742
+ int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
743
+ assert(head_size % thread_group_size == 0);
744
+
745
+ // NOTE: alibi_slopes is optional.
746
+ const float* alibi_slopes_ptr = alibi_slopes ?
747
+ reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
748
+ : nullptr;
749
+
750
+ T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
751
+ float* exp_sums_ptr = reinterpret_cast<float*>(exp_sums.data_ptr());
752
+ float* max_logits_ptr = reinterpret_cast<float*>(max_logits.data_ptr());
753
+ T* tmp_out_ptr = reinterpret_cast<T*>(tmp_out.data_ptr());
754
+ T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
755
+ T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
756
+ T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
757
+ int* head_mapping_ptr = reinterpret_cast<int*>(head_mapping.data_ptr());
758
+ int* block_tables_ptr = block_tables.data_ptr<int>();
759
+ int* context_lens_ptr = context_lens.data_ptr<int>();
760
+
761
+ constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
762
+ int max_num_partitions = DIVIDE_ROUND_UP(max_context_len, PARTITION_SIZE);
763
+ int logits_size = PARTITION_SIZE * sizeof(float);
764
+ int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
765
+
766
+ // For paged attention v2 kernel.
767
+ dim3 grid(num_heads, num_seqs, max_num_partitions);
768
+ int shared_mem_size = std::max(logits_size, outputs_size);
769
+ // For paged attention v2 reduce kernel.
770
+ dim3 reduce_grid(num_heads, num_seqs);
771
+ int reduce_shared_mem_size = 2 * max_num_partitions * sizeof(float);
772
+
773
dim3 block(NUM_THREADS);
774
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
775
switch (head_size) {
415
- // NOTE(woosuk): To reduce the compilation time, we omitted head sizes
416
- // 32, 160, 192.
417
- // case 32:
418
- // LAUNCH_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
419
- // break;
776
+ // NOTE(woosuk): To reduce the compilation time, we only compile for the
777
+ // head sizes that we use in the model. However, we can easily extend this
778
+ // to support any head size which is a multiple of 16.
779
case 64:
421
- LAUNCH_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
780
+ LAUNCH_PAGED_ATTENTION_V2(64);
781
break;
782
case 80:
424
- LAUNCH_ATTENTION_KERNEL(T, 80, BLOCK_SIZE, NUM_THREADS);
783
+ LAUNCH_PAGED_ATTENTION_V2(80);
784
break;
785
case 96:
427
- LAUNCH_ATTENTION_KERNEL(T, 96, BLOCK_SIZE, NUM_THREADS);
786
+ LAUNCH_PAGED_ATTENTION_V2(96);
787
break;
788
case 112:
430
- LAUNCH_ATTENTION_KERNEL(T, 112, BLOCK_SIZE, NUM_THREADS);
789
+ LAUNCH_PAGED_ATTENTION_V2(112);
790
break;
791
case 128:
433
- LAUNCH_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
792
+ LAUNCH_PAGED_ATTENTION_V2(128);
793
break;
435
- // case 160:
436
- // LAUNCH_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
437
- // break;
438
- // case 192:
439
- // LAUNCH_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
440
- // break;
794
case 256:
442
- LAUNCH_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
795
+ LAUNCH_PAGED_ATTENTION_V2(256);
796
break;
797
default:
798
TORCH_CHECK(false, "Unsupported head size: ", head_size);
@@ -447,9 +800,12 @@ void single_query_cached_kv_attention_launcher(
800
}
801
}
802
450
-#define CALL_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
451
- single_query_cached_kv_attention_launcher<T, BLOCK_SIZE>( \
803
+#define CALL_V2_LAUNCHER(T, BLOCK_SIZE) \
804
+ paged_attention_v2_launcher<T, BLOCK_SIZE>( \
805
out, \
806
+ exp_sums, \
807
+ max_logits, \
808
+ tmp_out, \
809
query, \
810
key_cache, \
811
value_cache, \
@@ -462,42 +818,27 @@ void single_query_cached_kv_attention_launcher(
818
819
// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
820
// 1, 2, 4, 64, 128, 256.
465
-#define CALL_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
821
+#define CALL_V2_LAUNCHER_BLOCK_SIZE(T) \
822
switch (block_size) { \
467
- /* case 1: */ \
468
- /* CALL_KERNEL_LAUNCHER(T, 1); */ \
469
- /* break; */ \
470
- /* case 2: */ \
471
- /* CALL_KERNEL_LAUNCHER(T, 2); */ \
472
- /* break; */ \
473
- /* case 4: */ \
474
- /* CALL_KERNEL_LAUNCHER(T, 4); */ \
475
- /* break; */ \
823
case 8: \
477
- CALL_KERNEL_LAUNCHER(T, 8); \
824
+ CALL_V2_LAUNCHER(T, 8); \
825
break; \
826
case 16: \
480
- CALL_KERNEL_LAUNCHER(T, 16); \
827
+ CALL_V2_LAUNCHER(T, 16); \
828
break; \
829
case 32: \
483
- CALL_KERNEL_LAUNCHER(T, 32); \
830
+ CALL_V2_LAUNCHER(T, 32); \
831
break; \
485
- /* case 64: */ \
486
- /* CALL_KERNEL_LAUNCHER(T, 64); */ \
487
- /* break; */ \
488
- /* case 128: */ \
489
- /* CALL_KERNEL_LAUNCHER(T, 128); */ \
490
- /* break; */ \
491
- /* case 256: */ \
492
- /* CALL_KERNEL_LAUNCHER(T, 256); */ \
493
- /* break; */ \
832
default: \
833
TORCH_CHECK(false, "Unsupported block size: ", block_size); \
834
break; \
835
}
836
499
-void single_query_cached_kv_attention(
837
+void paged_attention_v2(
838
torch::Tensor& out, // [num_seqs, num_heads, head_size]
839
+ torch::Tensor& exp_sums, // [num_seqs, num_heads, max_num_partitions]
840
+ torch::Tensor& max_logits, // [num_seqs, num_heads, max_num_partitions]
841
+ torch::Tensor& tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
842
torch::Tensor& query, // [num_seqs, num_heads, head_size]
843
torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
844
torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
@@ -509,11 +850,11 @@ void single_query_cached_kv_attention(
850
int max_context_len,
851
const c10::optional<torch::Tensor>& alibi_slopes) {
852
if (query.dtype() == at::ScalarType::Float) {
512
- CALL_KERNEL_LAUNCHER_BLOCK_SIZE(float);
853
+ CALL_V2_LAUNCHER_BLOCK_SIZE(float);
854
} else if (query.dtype() == at::ScalarType::Half) {
514
- CALL_KERNEL_LAUNCHER_BLOCK_SIZE(uint16_t);
855
+ CALL_V2_LAUNCHER_BLOCK_SIZE(uint16_t);
856
} else if (query.dtype() == at::ScalarType::BFloat16) {
516
- CALL_KERNEL_LAUNCHER_BLOCK_SIZE(__nv_bfloat16);
857
+ CALL_V2_LAUNCHER_BLOCK_SIZE(__nv_bfloat16);
858
} else {
859
TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
860
}
@@ -522,3 +863,4 @@ void single_query_cached_kv_attention(
863
#undef WARP_SIZE
864
#undef MAX
865
#undef MIN
866
+#undef DIVIDE_ROUND_UP
csrc/attention/dtype_bfloat16.cuh
+5
@@ -420,6 +420,11 @@ inline __device__ void from_float(bf16_8_t& dst, Float8_ src) {
420
#endif
421
}
422
423
+// From bfloat16 to float32.
424
+inline __device__ float to_float(__nv_bfloat16 u) {
425
+ return __bfloat162float(u);
426
+}
427
+
428
// Zero-out a variable.
429
inline __device__ void zero(__nv_bfloat16& dst) {
430
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
tests/kernels/test_attention.py
+54
-17
@@ -14,13 +14,14 @@ FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
14
# - 512 as a buffer
15
MAX_SEQ_LEN = get_max_shared_memory_bytes() // FLOAT32_BYTES - 512
16
NUM_BLOCKS = 128 # Arbitrary values for testing
17
+PARTITION_SIZE = 512
18
19
DTYPES = [torch.half, torch.bfloat16, torch.float]
20
NUM_GEN_SEQS = [7] # Arbitrary values for testing
20
-NUM_PREFILL_SEQS = [1, 3, 7] # Arbitrary values for testing
21
+NUM_PREFILL_SEQS = [3] # Arbitrary values for testing
22
NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
23
HEAD_SIZES = [64, 80, 96, 112, 128, 256]
23
-BLOCK_SIZES = [8, 16, 32]
24
+BLOCK_SIZES = [16, 32]
25
USE_ALIBI = [False, True]
26
SEEDS = [0]
27
@@ -96,6 +97,7 @@ def ref_single_query_cached_kv_attention(
97
output[i].copy_(out, non_blocking=True)
98
99
100
+@pytest.mark.parametrize("version", ["v1", "v2"])
101
@pytest.mark.parametrize("num_seqs", NUM_GEN_SEQS)
102
@pytest.mark.parametrize("num_heads", NUM_HEADS)
103
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@@ -103,9 +105,9 @@ def ref_single_query_cached_kv_attention(
105
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
106
@pytest.mark.parametrize("dtype", DTYPES)
107
@pytest.mark.parametrize("seed", SEEDS)
106
-@torch.inference_mode()
107
-def test_single_query_cached_kv_attention(
108
+def test_paged_attention(
109
kv_cache_factory,
110
+ version: str,
111
num_seqs: int,
112
num_heads: Tuple[int, int],
113
head_size: int,
@@ -162,19 +164,54 @@ def test_single_query_cached_kv_attention(
164
165
# Call the paged attention kernel.
166
output = torch.empty_like(query)
165
- attention_ops.single_query_cached_kv_attention(
166
- output,
167
- query,
168
- key_cache,
169
- value_cache,
170
- head_mapping,
171
- scale,
172
- block_tables,
173
- context_lens,
174
- block_size,
175
- max_context_len,
176
- alibi_slopes,
177
- )
167
+ if version == "v1":
168
+ attention_ops.paged_attention_v1(
169
+ output,
170
+ query,
171
+ key_cache,
172
+ value_cache,
173
+ head_mapping,
174
+ scale,
175
+ block_tables,
176
+ context_lens,
177
+ block_size,
178
+ max_context_len,
179
+ alibi_slopes,
180
+ )
181
+ elif version == "v2":
182
+ num_partitions = ((max_context_len + PARTITION_SIZE - 1) //
183
+ PARTITION_SIZE)
184
+ assert PARTITION_SIZE % block_size == 0
185
+ num_seqs, num_heads, head_size = output.shape
186
+ tmp_output = torch.empty(
187
+ size=(num_seqs, num_heads, num_partitions, head_size),
188
+ dtype=output.dtype,
189
+ device=output.device,
190
+ )
191
+ exp_sums = torch.empty(
192
+ size=(num_seqs, num_heads, num_partitions),
193
+ dtype=torch.float32,
194
+ device=output.device,
195
+ )
196
+ max_logits = torch.empty_like(exp_sums)
197
+ attention_ops.paged_attention_v2(
198
+ output,
199
+ exp_sums,
200
+ max_logits,
201
+ tmp_output,
202
+ query,
203
+ key_cache,
204
+ value_cache,
205
+ head_mapping,
206
+ scale,
207
+ block_tables,
208
+ context_lens,
209
+ block_size,
210
+ max_context_len,
211
+ alibi_slopes,
212
+ )
213
+ else:
214
+ assert False, f"Unknown version: {version}"
215
216
# Run the reference implementation.
217
ref_output = torch.empty_like(query)
vllm/model_executor/layers/attention.py
+71
-47
@@ -15,6 +15,8 @@ from vllm.model_executor.layers.rotary_embedding import (
15
RotaryEmbedding)
16
17
_SUPPORTED_HEAD_SIZES = [64, 80, 96, 112, 128, 256]
18
+# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
19
+_PARTITION_SIZE = 512
20
21
22
class PagedAttention(nn.Module):
@@ -130,6 +132,14 @@ class PagedAttention(nn.Module):
132
output.copy_(out.squeeze(0))
133
return output
134
135
+ def get_alibi_slopes(self) -> Optional[torch.Tensor]:
136
+ """Returns the slopes for the alibi attention bias.
137
+
138
+ Returns:
139
+ slopes: shape = [num_heads]
140
+ """
141
+ return None
142
+
143
def single_query_cached_kv_attention(
144
self,
145
output: torch.Tensor,
@@ -137,6 +147,7 @@ class PagedAttention(nn.Module):
147
key_cache: torch.Tensor,
148
value_cache: torch.Tensor,
149
input_metadata: InputMetadata,
150
+ alibi_slopes: Optional[torch.Tensor],
151
) -> None:
152
"""PagedAttention for the generation tokens.
153
@@ -148,21 +159,65 @@ class PagedAttention(nn.Module):
159
value_cache: shape = [num_blocks, num_kv_heads, head_size,
160
block_size]
161
input_metadata: metadata for paged attention.
162
+ alibi_slopes: shape = [num_heads]
163
"""
164
block_size = value_cache.shape[3]
153
- attention_ops.single_query_cached_kv_attention(
154
- output,
155
- query,
156
- key_cache,
157
- value_cache,
158
- self.head_mapping,
159
- self.scale,
160
- input_metadata.block_tables,
161
- input_metadata.context_lens,
162
- block_size,
163
- input_metadata.max_context_len,
164
- None, # alibi_slopes
165
- )
165
+ num_seqs, num_heads, head_size = query.shape
166
+ max_num_partitions = (
167
+ (input_metadata.max_context_len + _PARTITION_SIZE - 1) //
168
+ _PARTITION_SIZE)
169
+ # NOTE(woosuk): We use a simple heuristic to decide whether to use
170
+ # PagedAttention V1 or V2. If the number of partitions is 1, we use
171
+ # V1 to avoid the overhead of reduction. Also, if the number of
172
+ # sequences or heads is large, we use V1 since there is enough work
173
+ # to parallelize.
174
+ # TODO(woosuk): Tune this heuristic.
175
+ use_v1 = max_num_partitions == 1 or num_seqs * num_heads > 512
176
+ if use_v1:
177
+ # Run PagedAttention V1.
178
+ attention_ops.paged_attention_v1(
179
+ output,
180
+ query,
181
+ key_cache,
182
+ value_cache,
183
+ self.head_mapping,
184
+ self.scale,
185
+ input_metadata.block_tables,
186
+ input_metadata.context_lens,
187
+ block_size,
188
+ input_metadata.max_context_len,
189
+ alibi_slopes,
190
+ )
191
+ else:
192
+ # Run PagedAttention V2.
193
+ assert _PARTITION_SIZE % block_size == 0
194
+ tmp_output = torch.empty(
195
+ size=(num_seqs, num_heads, max_num_partitions, head_size),
196
+ dtype=output.dtype,
197
+ device=output.device,
198
+ )
199
+ exp_sums = torch.empty(
200
+ size=(num_seqs, num_heads, max_num_partitions),
201
+ dtype=torch.float32,
202
+ device=output.device,
203
+ )
204
+ max_logits = torch.empty_like(exp_sums)
205
+ attention_ops.paged_attention_v2(
206
+ output,
207
+ exp_sums,
208
+ max_logits,
209
+ tmp_output,
210
+ query,
211
+ key_cache,
212
+ value_cache,
213
+ self.head_mapping,
214
+ self.scale,
215
+ input_metadata.block_tables,
216
+ input_metadata.context_lens,
217
+ block_size,
218
+ input_metadata.max_context_len,
219
+ alibi_slopes,
220
+ )
221
222
def forward(
223
self,
@@ -253,7 +308,7 @@ class PagedAttention(nn.Module):
308
self.single_query_cached_kv_attention(
309
output[num_prompt_tokens:num_valid_tokens],
310
query[num_prompt_tokens:num_valid_tokens], key_cache,
256
- value_cache, input_metadata)
311
+ value_cache, input_metadata, self.get_alibi_slopes())
312
313
# Reshape the output tensor.
314
# NOTE(woosuk): The output tensor may include paddings.
@@ -431,36 +486,5 @@ class PagedAttentionWithALiBi(PagedAttention):
486
start += prompt_len
487
return output
488
434
- def single_query_cached_kv_attention(
435
- self,
436
- output: torch.Tensor,
437
- query: torch.Tensor,
438
- key_cache: torch.Tensor,
439
- value_cache: torch.Tensor,
440
- input_metadata: InputMetadata,
441
- ) -> None:
442
- """PagedAttention with ALiBi bias for the generation tokens.
443
-
444
- Args:
445
- output: shape = [num_generation_tokens, num_heads, head_size]
446
- query: shape = [num_generation_tokens, num_heads, head_size]
447
- key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
448
- block_size, x]
449
- value_cache: shape = [num_blocks, num_kv_heads, head_size,
450
- block_size]
451
- input_metadata: metadata for paged attention.
452
- """
453
- block_size = value_cache.shape[3]
454
- attention_ops.single_query_cached_kv_attention(
455
- output,
456
- query,
457
- key_cache,
458
- value_cache,
459
- self.head_mapping,
460
- self.scale,
461
- input_metadata.block_tables,
462
- input_metadata.context_lens,
463
- block_size,
464
- input_metadata.max_context_len,
465
- self.alibi_slopes,
466
- )
489
+ def get_alibi_slopes(self) -> Optional[torch.Tensor]:
490
+ return self.alibi_slopes