@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 928de46888
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Implement PagedAttention V2 (#1348)

Woosuk Kwon committed Oct 16, 2023 at 00:59 UTC 928de46888b9b257dfa491047a7d9cd199ca585b
6 files changed +764 -139
benchmarks/kernels/benchmark_paged_attention.py new
+197
@@ -0,0 +1,197 @@
1 +import argparse
2 +import random
3 +import time
4 +
5 +import torch
6 +
7 +from vllm import attention_ops
8 +
9 +NUM_BLOCKS = 1024
10 +PARTITION_SIZE = 512
11 +
12 +
13 +@torch.inference_mode()
14 +def main(
15 + version: str,
16 + num_seqs: int,
17 + context_len: int,
18 + num_query_heads: int,
19 + num_kv_heads: int,
20 + head_size: int,
21 + use_alibi: bool,
22 + block_size: int,
23 + dtype: torch.dtype,
24 + seed: int,
25 + do_profile: bool,
26 +) -> None:
27 + random.seed(seed)
28 + torch.random.manual_seed(seed)
29 + torch.cuda.manual_seed(seed)
30 +
31 + scale = float(1.0 / (head_size**0.5))
32 + query = torch.empty(num_seqs,
33 + num_query_heads,
34 + head_size,
35 + dtype=dtype,
36 + device="cuda")
37 + query.uniform_(-scale, scale)
38 +
39 + assert num_query_heads % num_kv_heads == 0
40 + num_queries_per_kv = num_query_heads // num_kv_heads
41 + head_mapping = torch.repeat_interleave(
42 + torch.arange(num_kv_heads, dtype=torch.int32, device="cuda"),
43 + num_queries_per_kv)
44 + alibi_slopes = None
45 + if use_alibi:
46 + alibi_slopes = torch.randn(num_query_heads,
47 + dtype=torch.float,
48 + device="cuda")
49 +
50 + context_lens = [context_len for _ in range(num_seqs)]
51 + max_context_len = max(context_lens)
52 + context_lens = torch.tensor(context_lens, dtype=torch.int, device="cuda")
53 +
54 + # Create the block tables.
55 + max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
56 + block_tables = []
57 + for _ in range(num_seqs):
58 + block_table = [
59 + random.randint(0, NUM_BLOCKS - 1)
60 + for _ in range(max_num_blocks_per_seq)
61 + ]
62 + block_tables.append(block_table)
63 + block_tables = torch.tensor(block_tables, dtype=torch.int, device="cuda")
64 +
65 + # Create the KV cache.
66 + x = 16 // torch.tensor([], dtype=dtype).element_size()
67 + key_cache_shape = (NUM_BLOCKS, num_kv_heads, head_size // x, block_size, x)
68 + key_cache = torch.empty(size=key_cache_shape, dtype=dtype, device="cuda")
69 + key_cache.uniform_(-scale, scale)
70 + value_cache_shape = (NUM_BLOCKS, num_kv_heads, head_size, block_size)
71 + value_cache = torch.empty(size=value_cache_shape,
72 + dtype=dtype,
73 + device="cuda")
74 + value_cache.uniform_(-scale, scale)
75 +
76 + # Prepare for the paged attention kernel.
77 + output = torch.empty_like(query)
78 + if version == "v2":
79 + num_partitions = ((max_context_len + PARTITION_SIZE - 1) //
80 + PARTITION_SIZE)
81 + tmp_output = torch.empty(
82 + size=(num_seqs, num_query_heads, num_partitions, head_size),
83 + dtype=output.dtype,
84 + device=output.device,
85 + )
86 + exp_sums = torch.empty(
87 + size=(num_seqs, num_query_heads, num_partitions),
88 + dtype=torch.float32,
89 + device=output.device,
90 + )
91 + max_logits = torch.empty_like(exp_sums)
92 +
93 + def run_benchmark(num_iters: int, profile: bool = False) -> float:
94 + torch.cuda.synchronize()
95 + if profile:
96 + torch.cuda.cudart().cudaProfilerStart()
97 + start_time = time.perf_counter()
98 +
99 + for _ in range(num_iters):
100 + if version == "v1":
101 + attention_ops.paged_attention_v1(
102 + output,
103 + query,
104 + key_cache,
105 + value_cache,
106 + head_mapping,
107 + scale,
108 + block_tables,
109 + context_lens,
110 + block_size,
111 + max_context_len,
112 + alibi_slopes,
113 + )
114 + elif version == "v2":
115 + attention_ops.paged_attention_v2(
116 + output,
117 + exp_sums,
118 + max_logits,
119 + tmp_output,
120 + query,
121 + key_cache,
122 + value_cache,
123 + head_mapping,
124 + scale,
125 + block_tables,
126 + context_lens,
127 + block_size,
128 + max_context_len,
129 + alibi_slopes,
130 + )
131 + else:
132 + raise ValueError(f"Invalid version: {version}")
133 + torch.cuda.synchronize()
134 +
135 + end_time = time.perf_counter()
136 + if profile:
137 + torch.cuda.cudart().cudaProfilerStart()
138 + return (end_time - start_time) / num_iters
139 +
140 + # Warmup.
141 + print("Warming up...")
142 + run_benchmark(num_iters=3, profile=False)
143 +
144 + # Benchmark.
145 + if do_profile:
146 + latency = run_benchmark(num_iters=1, profile=True)
147 + else:
148 + latency = run_benchmark(num_iters=100, profile=False)
149 + print(f"Kernel running time: {latency * 1000000:.3f} us")
150 +
151 +
152 +if __name__ == '__main__':
153 + parser = argparse.ArgumentParser(
154 + description="Benchmark the paged attention kernel.")
155 + parser.add_argument("--version",
156 + type=str,
157 + choices=["v1", "v2"],
158 + default="v2")
159 + parser.add_argument("--batch-size", type=int, default=8)
160 + parser.add_argument("--context-len", type=int, default=4096)
161 + parser.add_argument("--num-query-heads", type=int, default=64)
162 + parser.add_argument("--num-kv-heads", type=int, default=8)
163 + parser.add_argument("--head-size",
164 + type=int,
165 + choices=[64, 80, 96, 112, 128, 256],
166 + default=128)
167 + parser.add_argument("--block-size", type=int, choices=[16, 32], default=16)
168 + parser.add_argument("--use-alibi", action="store_true")
169 + parser.add_argument("--dtype",
170 + type=str,
171 + choices=["half", "bfloat16", "float"],
172 + default="half")
173 + parser.add_argument("--seed", type=int, default=0)
174 + parser.add_argument("--profile", action="store_true")
175 + args = parser.parse_args()
176 + print(args)
177 +
178 + if args.num_query_heads % args.num_kv_heads != 0:
179 + raise ValueError("num_query_heads must be divisible by num_kv_heads")
180 + dtype_to_torch_dtype = {
181 + "half": torch.half,
182 + "bfloat16": torch.bfloat16,
183 + "float": torch.float,
184 + }
185 + main(
186 + version=args.version,
187 + num_seqs=args.batch_size,
188 + context_len=args.context_len,
189 + num_query_heads=args.num_query_heads,
190 + num_kv_heads=args.num_kv_heads,
191 + head_size=args.head_size,
192 + block_size=args.block_size,
193 + use_alibi=args.use_alibi,
194 + dtype=dtype_to_torch_dtype[args.dtype],
195 + seed=args.seed,
196 + do_profile=args.profile,
197 + )
csrc/attention.cpp
+24 -4
@@ -1,7 +1,7 @@
1 #include <torch/extension.h>
2 #include <c10/util/Optional.h>
3
4 -void single_query_cached_kv_attention(
4 +void paged_attention_v1(
5 torch::Tensor& out,
6 torch::Tensor& query,
7 torch::Tensor& key_cache,
@@ -14,9 +14,29 @@ void single_query_cached_kv_attention(
14 int max_context_len,
15 const c10::optional<torch::Tensor>& alibi_slopes);
16
17 +void paged_attention_v2(
18 + torch::Tensor& out,
19 + torch::Tensor& exp_sums,
20 + torch::Tensor& max_logits,
21 + torch::Tensor& tmp_out,
22 + torch::Tensor& query,
23 + torch::Tensor& key_cache,
24 + torch::Tensor& value_cache,
25 + torch::Tensor& head_mapping,
26 + float scale,
27 + torch::Tensor& block_tables,
28 + torch::Tensor& context_lens,
29 + int block_size,
30 + int max_context_len,
31 + const c10::optional<torch::Tensor>& alibi_slopes);
32 +
33 PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
34 m.def(
19 - "single_query_cached_kv_attention",
20 - &single_query_cached_kv_attention,
21 - "Compute the attention between an input query and the cached key/value tensors");
35 + "paged_attention_v1",
36 + &paged_attention_v1,
37 + "Compute the attention between an input query and the cached keys/values using PagedAttention.");
38 + m.def(
39 + "paged_attention_v2",
40 + &paged_attention_v2,
41 + "PagedAttention V2.");
42 }
csrc/attention/attention_kernels.cu
+413 -71
@@ -26,6 +26,7 @@
26 #define WARP_SIZE 32
27 #define MAX(a, b) ((a) > (b) ? (a) : (b))
28 #define MIN(a, b) ((a) < (b) ? (a) : (b))
29 +#define DIVIDE_ROUND_UP(a, b) (((a) + (b) - 1) / (b))
30
31 namespace vllm {
32
@@ -65,14 +66,18 @@ inline __device__ float block_sum(float* red_smem, float sum) {
66 return __shfl_sync(uint32_t(-1), sum, 0);
67 }
68
68 -// Grid: (num_heads, num_seqs).
69 +// TODO(woosuk): Merge the last two dimensions of the grid.
70 +// Grid: (num_heads, num_seqs, max_num_partitions).
71 template<
72 typename scalar_t,
73 int HEAD_SIZE,
74 int BLOCK_SIZE,
73 - int NUM_THREADS>
74 -__global__ void single_query_cached_kv_attention_kernel(
75 - scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
75 + int NUM_THREADS,
76 + int PARTITION_SIZE = 0> // Zero means no partitioning.
77 +__device__ void paged_attention_kernel(
78 + float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
79 + float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
80 + scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions, head_size]
81 const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
82 const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
83 const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
@@ -85,10 +90,33 @@ __global__ void single_query_cached_kv_attention_kernel(
90 const int q_stride,
91 const int kv_block_stride,
92 const int kv_head_stride) {
93 + const int seq_idx = blockIdx.y;
94 + const int partition_idx = blockIdx.z;
95 + const int max_num_partitions = gridDim.z;
96 + constexpr bool USE_PARTITIONING = PARTITION_SIZE > 0;
97 + const int context_len = context_lens[seq_idx];
98 + if (USE_PARTITIONING && partition_idx * PARTITION_SIZE >= context_len) {
99 + // No work to do. Terminate the thread block.
100 + return;
101 + }
102 +
103 + const int num_context_blocks = DIVIDE_ROUND_UP(context_len, BLOCK_SIZE);
104 + const int num_blocks_per_partition = USE_PARTITIONING ? PARTITION_SIZE / BLOCK_SIZE : num_context_blocks;
105 +
106 + // [start_block_idx, end_block_idx) is the range of blocks to process.
107 + const int start_block_idx = USE_PARTITIONING ? partition_idx * num_blocks_per_partition : 0;
108 + const int end_block_idx = MIN(start_block_idx + num_blocks_per_partition, num_context_blocks);
109 + const int num_blocks = end_block_idx - start_block_idx;
110 +
111 + // [start_token_idx, end_token_idx) is the range of tokens to process.
112 + const int start_token_idx = start_block_idx * BLOCK_SIZE;
113 + const int end_token_idx = MIN(start_token_idx + num_blocks * BLOCK_SIZE, context_len);
114 + const int num_tokens = end_token_idx - start_token_idx;
115 +
116 constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
117 constexpr int NUM_THREAD_GROUPS = NUM_THREADS / THREAD_GROUP_SIZE; // Note: This assumes THREAD_GROUP_SIZE divides NUM_THREADS
118 assert(NUM_THREADS % THREAD_GROUP_SIZE == 0);
91 - constexpr int NUM_TOKENS_PER_THREAD_GROUP = (BLOCK_SIZE + WARP_SIZE - 1) / WARP_SIZE;
119 + constexpr int NUM_TOKENS_PER_THREAD_GROUP = DIVIDE_ROUND_UP(BLOCK_SIZE, WARP_SIZE);
120 constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
121 const int thread_idx = threadIdx.x;
122 const int warp_idx = thread_idx / WARP_SIZE;
@@ -97,7 +125,6 @@ __global__ void single_query_cached_kv_attention_kernel(
125 const int head_idx = blockIdx.x;
126 const int num_heads = gridDim.x;
127 const int kv_head_idx = head_mapping[head_idx];
100 - const int seq_idx = blockIdx.y;
128 const float alibi_slope = alibi_slopes == nullptr ? 0.f : alibi_slopes[head_idx];
129
130 // A vector type to store a part of a key or a query.
@@ -142,15 +169,12 @@ __global__ void single_query_cached_kv_attention_kernel(
169 constexpr int x = 16 / sizeof(scalar_t);
170 float qk_max = -FLT_MAX;
171
145 - const int* block_table = block_tables + seq_idx * max_num_blocks_per_seq;
146 - const int context_len = context_lens[seq_idx];
147 - const int num_blocks = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
148 -
172 // Iterate over the key blocks.
173 // Each warp fetches a block of keys for each iteration.
174 // Each thread group in a warp fetches a key from the block, and computes
175 // dot product with the query.
153 - for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
176 + const int* block_table = block_tables + seq_idx * max_num_blocks_per_seq;
177 + for (int block_idx = start_block_idx + warp_idx; block_idx < end_block_idx; block_idx += NUM_WARPS) {
178 const int physical_block_number = block_table[block_idx];
179
180 // Load a key to registers.
@@ -184,7 +208,7 @@ __global__ void single_query_cached_kv_attention_kernel(
208 // Store the partial reductions to shared memory.
209 // NOTE(woosuk): It is required to zero out the masked logits.
210 const bool mask = token_idx >= context_len;
187 - logits[token_idx] = mask ? 0.f : qk;
211 + logits[token_idx - start_token_idx] = mask ? 0.f : qk;
212 // Update the max value.
213 qk_max = mask ? qk_max : fmaxf(qk_max, qk);
214 }
@@ -215,7 +239,7 @@ __global__ void single_query_cached_kv_attention_kernel(
239
240 // Get the sum of the exp values.
241 float exp_sum = 0.f;
218 - for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
242 + for (int i = thread_idx; i < num_tokens; i += NUM_THREADS) {
243 float val = __expf(logits[i] - qk_max);
244 logits[i] = val;
245 exp_sum += val;
@@ -224,11 +248,23 @@ __global__ void single_query_cached_kv_attention_kernel(
248
249 // Compute softmax.
250 const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
227 - for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
251 + for (int i = thread_idx; i < num_tokens; i += NUM_THREADS) {
252 logits[i] *= inv_sum;
253 }
254 __syncthreads();
255
256 + // If partitioning is enabled, store the max logit and exp_sum.
257 + if (USE_PARTITIONING && thread_idx == 0) {
258 + float* max_logits_ptr = max_logits + seq_idx * num_heads * max_num_partitions
259 + + head_idx * max_num_partitions
260 + + partition_idx;
261 + *max_logits_ptr = qk_max;
262 + float* exp_sums_ptr = exp_sums + seq_idx * num_heads * max_num_partitions
263 + + head_idx * max_num_partitions
264 + + partition_idx;
265 + *exp_sums_ptr = exp_sum;
266 + }
267 +
268 // Each thread will fetch 16 bytes from the value cache at a time.
269 constexpr int V_VEC_SIZE = MIN(16 / sizeof(scalar_t), BLOCK_SIZE);
270 using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
@@ -237,7 +273,7 @@ __global__ void single_query_cached_kv_attention_kernel(
273
274 constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
275 constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
240 - constexpr int NUM_ROWS_PER_THREAD = (HEAD_SIZE + NUM_ROWS_PER_ITER - 1) / NUM_ROWS_PER_ITER;
276 + constexpr int NUM_ROWS_PER_THREAD = DIVIDE_ROUND_UP(HEAD_SIZE, NUM_ROWS_PER_ITER);
277
278 // NOTE(woosuk): We use FP32 for the accumulator for better accuracy.
279 float accs[NUM_ROWS_PER_THREAD];
@@ -248,12 +284,12 @@ __global__ void single_query_cached_kv_attention_kernel(
284
285 scalar_t zero_value;
286 zero(zero_value);
251 - for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
287 + for (int block_idx = start_block_idx + warp_idx; block_idx < end_block_idx; block_idx += NUM_WARPS) {
288 const int physical_block_number = block_table[block_idx];
289 const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
290 const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
291 L_vec logits_vec;
256 - from_float(logits_vec, *reinterpret_cast<Float_L_vec*>(logits + token_idx));
292 + from_float(logits_vec, *reinterpret_cast<Float_L_vec*>(logits + token_idx - start_token_idx));
293
294 const scalar_t* v_ptr = v_cache + physical_block_number * kv_block_stride
295 + kv_head_idx * kv_head_stride;
@@ -263,7 +299,7 @@ __global__ void single_query_cached_kv_attention_kernel(
299 if (row_idx < HEAD_SIZE) {
300 const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
301 V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
266 - if (block_idx == num_blocks - 1) {
302 + if (block_idx == num_context_blocks - 1) {
303 // NOTE(woosuk): When v_vec contains the tokens that are out of the context,
304 // we should explicitly zero out the values since they may contain NaNs.
305 // See https://github.com/vllm-project/vllm/issues/641#issuecomment-1682544472
@@ -327,7 +363,9 @@ __global__ void single_query_cached_kv_attention_kernel(
363
364 // Write the final output.
365 if (warp_idx == 0) {
330 - scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
366 + scalar_t* out_ptr = out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE
367 + + head_idx * max_num_partitions * HEAD_SIZE
368 + + partition_idx * HEAD_SIZE;
369 #pragma unroll
370 for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
371 const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
@@ -338,13 +376,167 @@ __global__ void single_query_cached_kv_attention_kernel(
376 }
377 }
378
379 +// Grid: (num_heads, num_seqs, 1).
380 +template<
381 + typename scalar_t,
382 + int HEAD_SIZE,
383 + int BLOCK_SIZE,
384 + int NUM_THREADS>
385 +__global__ void paged_attention_v1_kernel(
386 + scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
387 + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
388 + const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
389 + const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
390 + const int* __restrict__ head_mapping, // [num_heads]
391 + const float scale,
392 + const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
393 + const int* __restrict__ context_lens, // [num_seqs]
394 + const int max_num_blocks_per_seq,
395 + const float* __restrict__ alibi_slopes, // [num_heads]
396 + const int q_stride,
397 + const int kv_block_stride,
398 + const int kv_head_stride) {
399 + paged_attention_kernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>(
400 + /* exp_sums */ nullptr, /* max_logits */ nullptr,
401 + out, q, k_cache, v_cache, head_mapping, scale, block_tables, context_lens,
402 + max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride, kv_head_stride);
403 +}
404 +
405 +// Grid: (num_heads, num_seqs, max_num_partitions).
406 +template<
407 + typename scalar_t,
408 + int HEAD_SIZE,
409 + int BLOCK_SIZE,
410 + int NUM_THREADS,
411 + int PARTITION_SIZE>
412 +__global__ void paged_attention_v2_kernel(
413 + float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
414 + float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
415 + scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
416 + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
417 + const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
418 + const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
419 + const int* __restrict__ head_mapping, // [num_heads]
420 + const float scale,
421 + const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
422 + const int* __restrict__ context_lens, // [num_seqs]
423 + const int max_num_blocks_per_seq,
424 + const float* __restrict__ alibi_slopes, // [num_heads]
425 + const int q_stride,
426 + const int kv_block_stride,
427 + const int kv_head_stride) {
428 + paged_attention_kernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, PARTITION_SIZE>(
429 + exp_sums, max_logits, tmp_out, q, k_cache, v_cache, head_mapping, scale,
430 + block_tables, context_lens, max_num_blocks_per_seq, alibi_slopes,
431 + q_stride, kv_block_stride, kv_head_stride);
432 +}
433 +
434 +// Grid: (num_heads, num_seqs).
435 +template<
436 + typename scalar_t,
437 + int HEAD_SIZE,
438 + int NUM_THREADS,
439 + int PARTITION_SIZE>
440 +__global__ void paged_attention_v2_reduce_kernel(
441 + scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
442 + const float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
443 + const float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
444 + const scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
445 + const int* __restrict__ context_lens, // [num_seqs]
446 + const int max_num_partitions) {
447 + const int num_heads = gridDim.x;
448 + const int head_idx = blockIdx.x;
449 + const int seq_idx = blockIdx.y;
450 + const int context_len = context_lens[seq_idx];
451 + const int num_partitions = DIVIDE_ROUND_UP(context_len, PARTITION_SIZE);
452 + if (num_partitions == 1) {
453 + // No need to reduce. Only copy tmp_out to out.
454 + scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
455 + const scalar_t* tmp_out_ptr = tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE
456 + + head_idx * max_num_partitions * HEAD_SIZE;
457 + for (int i = threadIdx.x; i < HEAD_SIZE; i += blockDim.x) {
458 + out_ptr[i] = tmp_out_ptr[i];
459 + }
460 + // Terminate the thread block.
461 + return;
462 + }
463 +
464 + constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
465 + const int warp_idx = threadIdx.x / WARP_SIZE;
466 + const int lane = threadIdx.x % WARP_SIZE;
467 +
468 + // Size: 2 * num_partitions.
469 + extern __shared__ char shared_mem[];
470 + // Workspace for reduction.
471 + __shared__ float red_smem[2 * NUM_WARPS];
472 +
473 + // Load max logits to shared memory.
474 + float* shared_max_logits = reinterpret_cast<float*>(shared_mem);
475 + const float* max_logits_ptr = max_logits + seq_idx * num_heads * max_num_partitions
476 + + head_idx * max_num_partitions;
477 + float max_logit = -FLT_MAX;
478 + for (int i = threadIdx.x; i < num_partitions; i += blockDim.x) {
479 + const float l = max_logits_ptr[i];
480 + shared_max_logits[i] = l;
481 + max_logit = fmaxf(max_logit, l);
482 + }
483 + __syncthreads();
484 +
485 + // Get the global max logit.
486 + // Reduce within the warp.
487 +#pragma unroll
488 + for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
489 + max_logit = fmaxf(max_logit, __shfl_xor_sync(uint32_t(-1), max_logit, mask));
490 + }
491 + if (lane == 0) {
492 + red_smem[warp_idx] = max_logit;
493 + }
494 + __syncthreads();
495 + // Reduce across warps.
496 + max_logit = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
497 +#pragma unroll
498 + for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
499 + max_logit = fmaxf(max_logit, __shfl_xor_sync(uint32_t(-1), max_logit, mask));
500 + }
501 + // Broadcast the max value to all threads.
502 + max_logit = __shfl_sync(uint32_t(-1), max_logit, 0);
503 +
504 + // Load rescaled exp sums to shared memory.
505 + float* shared_exp_sums = reinterpret_cast<float*>(shared_mem + sizeof(float) * num_partitions);
506 + const float* exp_sums_ptr = exp_sums + seq_idx * num_heads * max_num_partitions
507 + + head_idx * max_num_partitions;
508 + float global_exp_sum = 0.0f;
509 + for (int i = threadIdx.x; i < num_partitions; i += blockDim.x) {
510 + float l = shared_max_logits[i];
511 + float rescaled_exp_sum = exp_sums_ptr[i] * expf(l - max_logit);
512 + global_exp_sum += rescaled_exp_sum;
513 + shared_exp_sums[i] = rescaled_exp_sum;
514 + }
515 + __syncthreads();
516 + global_exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], global_exp_sum);
517 + const float inv_global_exp_sum = __fdividef(1.0f, global_exp_sum + 1e-6f);
518 +
519 + // Aggregate tmp_out to out.
520 + const scalar_t* tmp_out_ptr = tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE
521 + + head_idx * max_num_partitions * HEAD_SIZE;
522 + scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
523 +#pragma unroll
524 + for (int i = threadIdx.x; i < HEAD_SIZE; i += NUM_THREADS) {
525 + float acc = 0.0f;
526 + for (int j = 0; j < num_partitions; ++j) {
527 + acc += to_float(tmp_out_ptr[j * HEAD_SIZE + i]) * shared_exp_sums[j] * inv_global_exp_sum;
528 + }
529 + from_float(out_ptr[i], acc);
530 + }
531 +}
532 +
533 } // namespace vllm
534
343 -#define LAUNCH_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
535 +#define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
536 cudaFuncSetAttribute( \
345 - vllm::single_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>, \
346 - cudaFuncAttributeMaxDynamicSharedMemorySize, shared_mem_size); \
347 - vllm::single_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
537 + vllm::paged_attention_v1_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>, \
538 + cudaFuncAttributeMaxDynamicSharedMemorySize, shared_mem_size); \
539 + vllm::paged_attention_v1_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
540 <<<grid, block, shared_mem_size, stream>>>( \
541 out_ptr, \
542 query_ptr, \
@@ -365,7 +557,7 @@ template<
557 typename T,
558 int BLOCK_SIZE,
559 int NUM_THREADS = 128>
368 -void single_query_cached_kv_attention_launcher(
560 +void paged_attention_v1_launcher(
561 torch::Tensor& out,
562 torch::Tensor& query,
563 torch::Tensor& key_cache,
@@ -401,45 +593,206 @@ void single_query_cached_kv_attention_launcher(
593 int* context_lens_ptr = context_lens.data_ptr<int>();
594
595 constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
404 - int padded_max_context_len = ((max_context_len + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
596 + int padded_max_context_len = DIVIDE_ROUND_UP(max_context_len, BLOCK_SIZE) * BLOCK_SIZE;
597 int logits_size = padded_max_context_len * sizeof(float);
598 int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
599 // Python-side check in vllm.worker.worker._check_if_can_support_max_seq_len
600 // Keep that in sync with the logic here!
601 int shared_mem_size = std::max(logits_size, outputs_size);
602
411 - dim3 grid(num_heads, num_seqs);
603 + dim3 grid(num_heads, num_seqs, 1);
604 + dim3 block(NUM_THREADS);
605 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
606 + switch (head_size) {
607 + // NOTE(woosuk): To reduce the compilation time, we only compile for the
608 + // head sizes that we use in the model. However, we can easily extend this
609 + // to support any head size which is a multiple of 16.
610 + case 64:
611 + LAUNCH_PAGED_ATTENTION_V1(64);
612 + break;
613 + case 80:
614 + LAUNCH_PAGED_ATTENTION_V1(80);
615 + break;
616 + case 96:
617 + LAUNCH_PAGED_ATTENTION_V1(96);
618 + break;
619 + case 112:
620 + LAUNCH_PAGED_ATTENTION_V1(112);
621 + break;
622 + case 128:
623 + LAUNCH_PAGED_ATTENTION_V1(128);
624 + break;
625 + case 256:
626 + LAUNCH_PAGED_ATTENTION_V1(256);
627 + break;
628 + default:
629 + TORCH_CHECK(false, "Unsupported head size: ", head_size);
630 + break;
631 + }
632 +}
633 +
634 +#define CALL_V1_LAUNCHER(T, BLOCK_SIZE) \
635 + paged_attention_v1_launcher<T, BLOCK_SIZE>( \
636 + out, \
637 + query, \
638 + key_cache, \
639 + value_cache, \
640 + head_mapping, \
641 + scale, \
642 + block_tables, \
643 + context_lens, \
644 + max_context_len, \
645 + alibi_slopes);
646 +
647 +// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
648 +// 1, 2, 4, 64, 128, 256.
649 +#define CALL_V1_LAUNCHER_BLOCK_SIZE(T) \
650 + switch (block_size) { \
651 + case 8: \
652 + CALL_V1_LAUNCHER(T, 8); \
653 + break; \
654 + case 16: \
655 + CALL_V1_LAUNCHER(T, 16); \
656 + break; \
657 + case 32: \
658 + CALL_V1_LAUNCHER(T, 32); \
659 + break; \
660 + default: \
661 + TORCH_CHECK(false, "Unsupported block size: ", block_size); \
662 + break; \
663 + }
664 +
665 +void paged_attention_v1(
666 + torch::Tensor& out, // [num_seqs, num_heads, head_size]
667 + torch::Tensor& query, // [num_seqs, num_heads, head_size]
668 + torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
669 + torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
670 + torch::Tensor& head_mapping, // [num_heads]
671 + float scale,
672 + torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
673 + torch::Tensor& context_lens, // [num_seqs]
674 + int block_size,
675 + int max_context_len,
676 + const c10::optional<torch::Tensor>& alibi_slopes) {
677 + if (query.dtype() == at::ScalarType::Float) {
678 + CALL_V1_LAUNCHER_BLOCK_SIZE(float);
679 + } else if (query.dtype() == at::ScalarType::Half) {
680 + CALL_V1_LAUNCHER_BLOCK_SIZE(uint16_t);
681 + } else if (query.dtype() == at::ScalarType::BFloat16) {
682 + CALL_V1_LAUNCHER_BLOCK_SIZE(__nv_bfloat16);
683 + } else {
684 + TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
685 + }
686 +}
687 +
688 +#define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
689 + vllm::paged_attention_v2_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, PARTITION_SIZE> \
690 + <<<grid, block, shared_mem_size, stream>>>( \
691 + exp_sums_ptr, \
692 + max_logits_ptr, \
693 + tmp_out_ptr, \
694 + query_ptr, \
695 + key_cache_ptr, \
696 + value_cache_ptr, \
697 + head_mapping_ptr, \
698 + scale, \
699 + block_tables_ptr, \
700 + context_lens_ptr, \
701 + max_num_blocks_per_seq, \
702 + alibi_slopes_ptr, \
703 + q_stride, \
704 + kv_block_stride, \
705 + kv_head_stride); \
706 + vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, PARTITION_SIZE> \
707 + <<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
708 + out_ptr, \
709 + exp_sums_ptr, \
710 + max_logits_ptr, \
711 + tmp_out_ptr, \
712 + context_lens_ptr, \
713 + max_num_partitions);
714 +
715 +template<
716 + typename T,
717 + int BLOCK_SIZE,
718 + int NUM_THREADS = 128,
719 + int PARTITION_SIZE = 512>
720 +void paged_attention_v2_launcher(
721 + torch::Tensor& out,
722 + torch::Tensor& exp_sums,
723 + torch::Tensor& max_logits,
724 + torch::Tensor& tmp_out,
725 + torch::Tensor& query,
726 + torch::Tensor& key_cache,
727 + torch::Tensor& value_cache,
728 + torch::Tensor& head_mapping,
729 + float scale,
730 + torch::Tensor& block_tables,
731 + torch::Tensor& context_lens,
732 + int max_context_len,
733 + const c10::optional<torch::Tensor>& alibi_slopes) {
734 + int num_seqs = query.size(0);
735 + int num_heads = query.size(1);
736 + int head_size = query.size(2);
737 + int max_num_blocks_per_seq = block_tables.size(1);
738 + int q_stride = query.stride(0);
739 + int kv_block_stride = key_cache.stride(0);
740 + int kv_head_stride = key_cache.stride(1);
741 +
742 + int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
743 + assert(head_size % thread_group_size == 0);
744 +
745 + // NOTE: alibi_slopes is optional.
746 + const float* alibi_slopes_ptr = alibi_slopes ?
747 + reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
748 + : nullptr;
749 +
750 + T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
751 + float* exp_sums_ptr = reinterpret_cast<float*>(exp_sums.data_ptr());
752 + float* max_logits_ptr = reinterpret_cast<float*>(max_logits.data_ptr());
753 + T* tmp_out_ptr = reinterpret_cast<T*>(tmp_out.data_ptr());
754 + T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
755 + T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
756 + T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
757 + int* head_mapping_ptr = reinterpret_cast<int*>(head_mapping.data_ptr());
758 + int* block_tables_ptr = block_tables.data_ptr<int>();
759 + int* context_lens_ptr = context_lens.data_ptr<int>();
760 +
761 + constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
762 + int max_num_partitions = DIVIDE_ROUND_UP(max_context_len, PARTITION_SIZE);
763 + int logits_size = PARTITION_SIZE * sizeof(float);
764 + int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
765 +
766 + // For paged attention v2 kernel.
767 + dim3 grid(num_heads, num_seqs, max_num_partitions);
768 + int shared_mem_size = std::max(logits_size, outputs_size);
769 + // For paged attention v2 reduce kernel.
770 + dim3 reduce_grid(num_heads, num_seqs);
771 + int reduce_shared_mem_size = 2 * max_num_partitions * sizeof(float);
772 +
773 dim3 block(NUM_THREADS);
774 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
775 switch (head_size) {
415 - // NOTE(woosuk): To reduce the compilation time, we omitted head sizes
416 - // 32, 160, 192.
417 - // case 32:
418 - // LAUNCH_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
419 - // break;
776 + // NOTE(woosuk): To reduce the compilation time, we only compile for the
777 + // head sizes that we use in the model. However, we can easily extend this
778 + // to support any head size which is a multiple of 16.
779 case 64:
421 - LAUNCH_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
780 + LAUNCH_PAGED_ATTENTION_V2(64);
781 break;
782 case 80:
424 - LAUNCH_ATTENTION_KERNEL(T, 80, BLOCK_SIZE, NUM_THREADS);
783 + LAUNCH_PAGED_ATTENTION_V2(80);
784 break;
785 case 96:
427 - LAUNCH_ATTENTION_KERNEL(T, 96, BLOCK_SIZE, NUM_THREADS);
786 + LAUNCH_PAGED_ATTENTION_V2(96);
787 break;
788 case 112:
430 - LAUNCH_ATTENTION_KERNEL(T, 112, BLOCK_SIZE, NUM_THREADS);
789 + LAUNCH_PAGED_ATTENTION_V2(112);
790 break;
791 case 128:
433 - LAUNCH_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
792 + LAUNCH_PAGED_ATTENTION_V2(128);
793 break;
435 - // case 160:
436 - // LAUNCH_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
437 - // break;
438 - // case 192:
439 - // LAUNCH_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
440 - // break;
794 case 256:
442 - LAUNCH_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
795 + LAUNCH_PAGED_ATTENTION_V2(256);
796 break;
797 default:
798 TORCH_CHECK(false, "Unsupported head size: ", head_size);
@@ -447,9 +800,12 @@ void single_query_cached_kv_attention_launcher(
800 }
801 }
802
450 -#define CALL_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
451 - single_query_cached_kv_attention_launcher<T, BLOCK_SIZE>( \
803 +#define CALL_V2_LAUNCHER(T, BLOCK_SIZE) \
804 + paged_attention_v2_launcher<T, BLOCK_SIZE>( \
805 out, \
806 + exp_sums, \
807 + max_logits, \
808 + tmp_out, \
809 query, \
810 key_cache, \
811 value_cache, \
@@ -462,42 +818,27 @@ void single_query_cached_kv_attention_launcher(
818
819 // NOTE(woosuk): To reduce the compilation time, we omitted block sizes
820 // 1, 2, 4, 64, 128, 256.
465 -#define CALL_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
821 +#define CALL_V2_LAUNCHER_BLOCK_SIZE(T) \
822 switch (block_size) { \
467 - /* case 1: */ \
468 - /* CALL_KERNEL_LAUNCHER(T, 1); */ \
469 - /* break; */ \
470 - /* case 2: */ \
471 - /* CALL_KERNEL_LAUNCHER(T, 2); */ \
472 - /* break; */ \
473 - /* case 4: */ \
474 - /* CALL_KERNEL_LAUNCHER(T, 4); */ \
475 - /* break; */ \
823 case 8: \
477 - CALL_KERNEL_LAUNCHER(T, 8); \
824 + CALL_V2_LAUNCHER(T, 8); \
825 break; \
826 case 16: \
480 - CALL_KERNEL_LAUNCHER(T, 16); \
827 + CALL_V2_LAUNCHER(T, 16); \
828 break; \
829 case 32: \
483 - CALL_KERNEL_LAUNCHER(T, 32); \
830 + CALL_V2_LAUNCHER(T, 32); \
831 break; \
485 - /* case 64: */ \
486 - /* CALL_KERNEL_LAUNCHER(T, 64); */ \
487 - /* break; */ \
488 - /* case 128: */ \
489 - /* CALL_KERNEL_LAUNCHER(T, 128); */ \
490 - /* break; */ \
491 - /* case 256: */ \
492 - /* CALL_KERNEL_LAUNCHER(T, 256); */ \
493 - /* break; */ \
832 default: \
833 TORCH_CHECK(false, "Unsupported block size: ", block_size); \
834 break; \
835 }
836
499 -void single_query_cached_kv_attention(
837 +void paged_attention_v2(
838 torch::Tensor& out, // [num_seqs, num_heads, head_size]
839 + torch::Tensor& exp_sums, // [num_seqs, num_heads, max_num_partitions]
840 + torch::Tensor& max_logits, // [num_seqs, num_heads, max_num_partitions]
841 + torch::Tensor& tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
842 torch::Tensor& query, // [num_seqs, num_heads, head_size]
843 torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
844 torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
@@ -509,11 +850,11 @@ void single_query_cached_kv_attention(
850 int max_context_len,
851 const c10::optional<torch::Tensor>& alibi_slopes) {
852 if (query.dtype() == at::ScalarType::Float) {
512 - CALL_KERNEL_LAUNCHER_BLOCK_SIZE(float);
853 + CALL_V2_LAUNCHER_BLOCK_SIZE(float);
854 } else if (query.dtype() == at::ScalarType::Half) {
514 - CALL_KERNEL_LAUNCHER_BLOCK_SIZE(uint16_t);
855 + CALL_V2_LAUNCHER_BLOCK_SIZE(uint16_t);
856 } else if (query.dtype() == at::ScalarType::BFloat16) {
516 - CALL_KERNEL_LAUNCHER_BLOCK_SIZE(__nv_bfloat16);
857 + CALL_V2_LAUNCHER_BLOCK_SIZE(__nv_bfloat16);
858 } else {
859 TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
860 }
@@ -522,3 +863,4 @@ void single_query_cached_kv_attention(
863 #undef WARP_SIZE
864 #undef MAX
865 #undef MIN
866 +#undef DIVIDE_ROUND_UP
csrc/attention/dtype_bfloat16.cuh
+5
@@ -420,6 +420,11 @@ inline __device__ void from_float(bf16_8_t& dst, Float8_ src) {
420 #endif
421 }
422
423 +// From bfloat16 to float32.
424 +inline __device__ float to_float(__nv_bfloat16 u) {
425 + return __bfloat162float(u);
426 +}
427 +
428 // Zero-out a variable.
429 inline __device__ void zero(__nv_bfloat16& dst) {
430 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
tests/kernels/test_attention.py
+54 -17
@@ -14,13 +14,14 @@ FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
14 # - 512 as a buffer
15 MAX_SEQ_LEN = get_max_shared_memory_bytes() // FLOAT32_BYTES - 512
16 NUM_BLOCKS = 128 # Arbitrary values for testing
17 +PARTITION_SIZE = 512
18
19 DTYPES = [torch.half, torch.bfloat16, torch.float]
20 NUM_GEN_SEQS = [7] # Arbitrary values for testing
20 -NUM_PREFILL_SEQS = [1, 3, 7] # Arbitrary values for testing
21 +NUM_PREFILL_SEQS = [3] # Arbitrary values for testing
22 NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
23 HEAD_SIZES = [64, 80, 96, 112, 128, 256]
23 -BLOCK_SIZES = [8, 16, 32]
24 +BLOCK_SIZES = [16, 32]
25 USE_ALIBI = [False, True]
26 SEEDS = [0]
27
@@ -96,6 +97,7 @@ def ref_single_query_cached_kv_attention(
97 output[i].copy_(out, non_blocking=True)
98
99
100 +@pytest.mark.parametrize("version", ["v1", "v2"])
101 @pytest.mark.parametrize("num_seqs", NUM_GEN_SEQS)
102 @pytest.mark.parametrize("num_heads", NUM_HEADS)
103 @pytest.mark.parametrize("head_size", HEAD_SIZES)
@@ -103,9 +105,9 @@ def ref_single_query_cached_kv_attention(
105 @pytest.mark.parametrize("block_size", BLOCK_SIZES)
106 @pytest.mark.parametrize("dtype", DTYPES)
107 @pytest.mark.parametrize("seed", SEEDS)
106 -@torch.inference_mode()
107 -def test_single_query_cached_kv_attention(
108 +def test_paged_attention(
109 kv_cache_factory,
110 + version: str,
111 num_seqs: int,
112 num_heads: Tuple[int, int],
113 head_size: int,
@@ -162,19 +164,54 @@ def test_single_query_cached_kv_attention(
164
165 # Call the paged attention kernel.
166 output = torch.empty_like(query)
165 - attention_ops.single_query_cached_kv_attention(
166 - output,
167 - query,
168 - key_cache,
169 - value_cache,
170 - head_mapping,
171 - scale,
172 - block_tables,
173 - context_lens,
174 - block_size,
175 - max_context_len,
176 - alibi_slopes,
177 - )
167 + if version == "v1":
168 + attention_ops.paged_attention_v1(
169 + output,
170 + query,
171 + key_cache,
172 + value_cache,
173 + head_mapping,
174 + scale,
175 + block_tables,
176 + context_lens,
177 + block_size,
178 + max_context_len,
179 + alibi_slopes,
180 + )
181 + elif version == "v2":
182 + num_partitions = ((max_context_len + PARTITION_SIZE - 1) //
183 + PARTITION_SIZE)
184 + assert PARTITION_SIZE % block_size == 0
185 + num_seqs, num_heads, head_size = output.shape
186 + tmp_output = torch.empty(
187 + size=(num_seqs, num_heads, num_partitions, head_size),
188 + dtype=output.dtype,
189 + device=output.device,
190 + )
191 + exp_sums = torch.empty(
192 + size=(num_seqs, num_heads, num_partitions),
193 + dtype=torch.float32,
194 + device=output.device,
195 + )
196 + max_logits = torch.empty_like(exp_sums)
197 + attention_ops.paged_attention_v2(
198 + output,
199 + exp_sums,
200 + max_logits,
201 + tmp_output,
202 + query,
203 + key_cache,
204 + value_cache,
205 + head_mapping,
206 + scale,
207 + block_tables,
208 + context_lens,
209 + block_size,
210 + max_context_len,
211 + alibi_slopes,
212 + )
213 + else:
214 + assert False, f"Unknown version: {version}"
215
216 # Run the reference implementation.
217 ref_output = torch.empty_like(query)
vllm/model_executor/layers/attention.py
+71 -47
@@ -15,6 +15,8 @@ from vllm.model_executor.layers.rotary_embedding import (
15 RotaryEmbedding)
16
17 _SUPPORTED_HEAD_SIZES = [64, 80, 96, 112, 128, 256]
18 +# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
19 +_PARTITION_SIZE = 512
20
21
22 class PagedAttention(nn.Module):
@@ -130,6 +132,14 @@ class PagedAttention(nn.Module):
132 output.copy_(out.squeeze(0))
133 return output
134
135 + def get_alibi_slopes(self) -> Optional[torch.Tensor]:
136 + """Returns the slopes for the alibi attention bias.
137 +
138 + Returns:
139 + slopes: shape = [num_heads]
140 + """
141 + return None
142 +
143 def single_query_cached_kv_attention(
144 self,
145 output: torch.Tensor,
@@ -137,6 +147,7 @@ class PagedAttention(nn.Module):
147 key_cache: torch.Tensor,
148 value_cache: torch.Tensor,
149 input_metadata: InputMetadata,
150 + alibi_slopes: Optional[torch.Tensor],
151 ) -> None:
152 """PagedAttention for the generation tokens.
153
@@ -148,21 +159,65 @@ class PagedAttention(nn.Module):
159 value_cache: shape = [num_blocks, num_kv_heads, head_size,
160 block_size]
161 input_metadata: metadata for paged attention.
162 + alibi_slopes: shape = [num_heads]
163 """
164 block_size = value_cache.shape[3]
153 - attention_ops.single_query_cached_kv_attention(
154 - output,
155 - query,
156 - key_cache,
157 - value_cache,
158 - self.head_mapping,
159 - self.scale,
160 - input_metadata.block_tables,
161 - input_metadata.context_lens,
162 - block_size,
163 - input_metadata.max_context_len,
164 - None, # alibi_slopes
165 - )
165 + num_seqs, num_heads, head_size = query.shape
166 + max_num_partitions = (
167 + (input_metadata.max_context_len + _PARTITION_SIZE - 1) //
168 + _PARTITION_SIZE)
169 + # NOTE(woosuk): We use a simple heuristic to decide whether to use
170 + # PagedAttention V1 or V2. If the number of partitions is 1, we use
171 + # V1 to avoid the overhead of reduction. Also, if the number of
172 + # sequences or heads is large, we use V1 since there is enough work
173 + # to parallelize.
174 + # TODO(woosuk): Tune this heuristic.
175 + use_v1 = max_num_partitions == 1 or num_seqs * num_heads > 512
176 + if use_v1:
177 + # Run PagedAttention V1.
178 + attention_ops.paged_attention_v1(
179 + output,
180 + query,
181 + key_cache,
182 + value_cache,
183 + self.head_mapping,
184 + self.scale,
185 + input_metadata.block_tables,
186 + input_metadata.context_lens,
187 + block_size,
188 + input_metadata.max_context_len,
189 + alibi_slopes,
190 + )
191 + else:
192 + # Run PagedAttention V2.
193 + assert _PARTITION_SIZE % block_size == 0
194 + tmp_output = torch.empty(
195 + size=(num_seqs, num_heads, max_num_partitions, head_size),
196 + dtype=output.dtype,
197 + device=output.device,
198 + )
199 + exp_sums = torch.empty(
200 + size=(num_seqs, num_heads, max_num_partitions),
201 + dtype=torch.float32,
202 + device=output.device,
203 + )
204 + max_logits = torch.empty_like(exp_sums)
205 + attention_ops.paged_attention_v2(
206 + output,
207 + exp_sums,
208 + max_logits,
209 + tmp_output,
210 + query,
211 + key_cache,
212 + value_cache,
213 + self.head_mapping,
214 + self.scale,
215 + input_metadata.block_tables,
216 + input_metadata.context_lens,
217 + block_size,
218 + input_metadata.max_context_len,
219 + alibi_slopes,
220 + )
221
222 def forward(
223 self,
@@ -253,7 +308,7 @@ class PagedAttention(nn.Module):
308 self.single_query_cached_kv_attention(
309 output[num_prompt_tokens:num_valid_tokens],
310 query[num_prompt_tokens:num_valid_tokens], key_cache,
256 - value_cache, input_metadata)
311 + value_cache, input_metadata, self.get_alibi_slopes())
312
313 # Reshape the output tensor.
314 # NOTE(woosuk): The output tensor may include paddings.
@@ -431,36 +486,5 @@ class PagedAttentionWithALiBi(PagedAttention):
486 start += prompt_len
487 return output
488
434 - def single_query_cached_kv_attention(
435 - self,
436 - output: torch.Tensor,
437 - query: torch.Tensor,
438 - key_cache: torch.Tensor,
439 - value_cache: torch.Tensor,
440 - input_metadata: InputMetadata,
441 - ) -> None:
442 - """PagedAttention with ALiBi bias for the generation tokens.
443 -
444 - Args:
445 - output: shape = [num_generation_tokens, num_heads, head_size]
446 - query: shape = [num_generation_tokens, num_heads, head_size]
447 - key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
448 - block_size, x]
449 - value_cache: shape = [num_blocks, num_kv_heads, head_size,
450 - block_size]
451 - input_metadata: metadata for paged attention.
452 - """
453 - block_size = value_cache.shape[3]
454 - attention_ops.single_query_cached_kv_attention(
455 - output,
456 - query,
457 - key_cache,
458 - value_cache,
459 - self.head_mapping,
460 - self.scale,
461 - input_metadata.block_tables,
462 - input_metadata.context_lens,
463 - block_size,
464 - input_metadata.max_context_len,
465 - self.alibi_slopes,
466 - )
489 + def get_alibi_slopes(self) -> Optional[torch.Tensor]:
490 + return self.alibi_slopes