A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Replace head_mapping params with num_kv_heads to attention kernel. (#1997)
Co-authored-by: wangguoya <wangguoya@baidu.com> Co-authored-by: Yang Zhao <zhaoyangstar@foxmail.com>
wbn committed
Dec 11, 2023 at 02:12 UTC
dacaf5a40056c40be4a84f0aec42278335f76dae
5 files changed
+26
-37
benchmarks/kernels/benchmark_paged_attention.py
+2
-6
@@ -37,10 +37,6 @@ def main(
37
query.uniform_(-scale, scale)
38
39
assert num_query_heads % num_kv_heads == 0
40
- num_queries_per_kv = num_query_heads // num_kv_heads
41
- head_mapping = torch.repeat_interleave(
42
- torch.arange(num_kv_heads, dtype=torch.int32, device="cuda"),
43
- num_queries_per_kv)
40
alibi_slopes = None
41
if use_alibi:
42
alibi_slopes = torch.randn(num_query_heads,
@@ -103,7 +99,7 @@ def main(
99
query,
100
key_cache,
101
value_cache,
106
- head_mapping,
102
+ num_kv_heads,
103
scale,
104
block_tables,
105
context_lens,
@@ -120,7 +116,7 @@ def main(
116
query,
117
key_cache,
118
value_cache,
123
- head_mapping,
119
+ num_kv_heads,
120
scale,
121
block_tables,
122
context_lens,
csrc/attention/attention_kernels.cu
+15
-16
@@ -89,7 +89,7 @@ __device__ void paged_attention_kernel(
89
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
90
const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
91
const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
92
- const int* __restrict__ head_mapping, // [num_heads]
92
+ const int num_kv_heads, // [num_heads]
93
const float scale,
94
const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
95
const int* __restrict__ context_lens, // [num_seqs]
@@ -132,7 +132,8 @@ __device__ void paged_attention_kernel(
132
133
const int head_idx = blockIdx.x;
134
const int num_heads = gridDim.x;
135
- const int kv_head_idx = head_mapping[head_idx];
135
+ const int num_queries_per_kv = num_heads / num_kv_heads;
136
+ const int kv_head_idx = head_idx / num_queries_per_kv;
137
const float alibi_slope = alibi_slopes == nullptr ? 0.f : alibi_slopes[head_idx];
138
139
// A vector type to store a part of a key or a query.
@@ -401,7 +402,7 @@ __global__ void paged_attention_v1_kernel(
402
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
403
const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
404
const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
404
- const int* __restrict__ head_mapping, // [num_heads]
405
+ const int num_kv_heads, // [num_heads]
406
const float scale,
407
const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
408
const int* __restrict__ context_lens, // [num_seqs]
@@ -412,7 +413,7 @@ __global__ void paged_attention_v1_kernel(
413
const int kv_head_stride) {
414
paged_attention_kernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>(
415
/* exp_sums */ nullptr, /* max_logits */ nullptr,
415
- out, q, k_cache, v_cache, head_mapping, scale, block_tables, context_lens,
416
+ out, q, k_cache, v_cache, num_kv_heads, scale, block_tables, context_lens,
417
max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride, kv_head_stride);
418
}
419
@@ -430,7 +431,7 @@ __global__ void paged_attention_v2_kernel(
431
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
432
const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
433
const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
433
- const int* __restrict__ head_mapping, // [num_heads]
434
+ const int num_kv_heads, // [num_heads]
435
const float scale,
436
const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
437
const int* __restrict__ context_lens, // [num_seqs]
@@ -440,7 +441,7 @@ __global__ void paged_attention_v2_kernel(
441
const int kv_block_stride,
442
const int kv_head_stride) {
443
paged_attention_kernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, PARTITION_SIZE>(
443
- exp_sums, max_logits, tmp_out, q, k_cache, v_cache, head_mapping, scale,
444
+ exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
445
block_tables, context_lens, max_num_blocks_per_seq, alibi_slopes,
446
q_stride, kv_block_stride, kv_head_stride);
447
}
@@ -556,7 +557,7 @@ __global__ void paged_attention_v2_reduce_kernel(
557
query_ptr, \
558
key_cache_ptr, \
559
value_cache_ptr, \
559
- head_mapping_ptr, \
560
+ num_kv_heads, \
561
scale, \
562
block_tables_ptr, \
563
context_lens_ptr, \
@@ -576,7 +577,7 @@ void paged_attention_v1_launcher(
577
torch::Tensor& query,
578
torch::Tensor& key_cache,
579
torch::Tensor& value_cache,
579
- torch::Tensor& head_mapping,
580
+ int num_kv_heads,
581
float scale,
582
torch::Tensor& block_tables,
583
torch::Tensor& context_lens,
@@ -602,7 +603,6 @@ void paged_attention_v1_launcher(
603
T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
604
T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
605
T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
605
- int* head_mapping_ptr = reinterpret_cast<int*>(head_mapping.data_ptr());
606
int* block_tables_ptr = block_tables.data_ptr<int>();
607
int* context_lens_ptr = context_lens.data_ptr<int>();
608
@@ -651,7 +651,7 @@ void paged_attention_v1_launcher(
651
query, \
652
key_cache, \
653
value_cache, \
654
- head_mapping, \
654
+ num_kv_heads, \
655
scale, \
656
block_tables, \
657
context_lens, \
@@ -681,7 +681,7 @@ void paged_attention_v1(
681
torch::Tensor& query, // [num_seqs, num_heads, head_size]
682
torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
683
torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
684
- torch::Tensor& head_mapping, // [num_heads]
684
+ int num_kv_heads, // [num_heads]
685
float scale,
686
torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
687
torch::Tensor& context_lens, // [num_seqs]
@@ -708,7 +708,7 @@ void paged_attention_v1(
708
query_ptr, \
709
key_cache_ptr, \
710
value_cache_ptr, \
711
- head_mapping_ptr, \
711
+ num_kv_heads, \
712
scale, \
713
block_tables_ptr, \
714
context_lens_ptr, \
@@ -739,7 +739,7 @@ void paged_attention_v2_launcher(
739
torch::Tensor& query,
740
torch::Tensor& key_cache,
741
torch::Tensor& value_cache,
742
- torch::Tensor& head_mapping,
742
+ int num_kv_heads,
743
float scale,
744
torch::Tensor& block_tables,
745
torch::Tensor& context_lens,
@@ -768,7 +768,6 @@ void paged_attention_v2_launcher(
768
T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
769
T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
770
T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
771
- int* head_mapping_ptr = reinterpret_cast<int*>(head_mapping.data_ptr());
771
int* block_tables_ptr = block_tables.data_ptr<int>();
772
int* context_lens_ptr = context_lens.data_ptr<int>();
773
@@ -823,7 +822,7 @@ void paged_attention_v2_launcher(
822
query, \
823
key_cache, \
824
value_cache, \
826
- head_mapping, \
825
+ num_kv_heads, \
826
scale, \
827
block_tables, \
828
context_lens, \
@@ -856,7 +855,7 @@ void paged_attention_v2(
855
torch::Tensor& query, // [num_seqs, num_heads, head_size]
856
torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
857
torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
859
- torch::Tensor& head_mapping, // [num_heads]
858
+ int num_kv_heads, // [num_heads]
859
float scale,
860
torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
861
torch::Tensor& context_lens, // [num_seqs]
csrc/ops.h
+2
-2
@@ -5,7 +5,7 @@ void paged_attention_v1(
5
torch::Tensor& query,
6
torch::Tensor& key_cache,
7
torch::Tensor& value_cache,
8
- torch::Tensor& head_mapping,
8
+ int num_kv_heads,
9
float scale,
10
torch::Tensor& block_tables,
11
torch::Tensor& context_lens,
@@ -21,7 +21,7 @@ void paged_attention_v2(
21
torch::Tensor& query,
22
torch::Tensor& key_cache,
23
torch::Tensor& value_cache,
24
- torch::Tensor& head_mapping,
24
+ int num_kv_heads,
25
float scale,
26
torch::Tensor& block_tables,
27
torch::Tensor& context_lens,
tests/kernels/test_attention.py
+2
-5
@@ -131,9 +131,6 @@ def test_paged_attention(
131
132
assert num_query_heads % num_kv_heads == 0
133
num_queries_per_kv = num_query_heads // num_kv_heads
134
- head_mapping = torch.repeat_interleave(
135
- torch.arange(num_kv_heads, dtype=torch.int32, device="cuda"),
136
- num_queries_per_kv)
134
alibi_slopes = None
135
if use_alibi:
136
alibi_slopes = torch.randn(num_query_heads,
@@ -170,7 +167,7 @@ def test_paged_attention(
167
query,
168
key_cache,
169
value_cache,
173
- head_mapping,
170
+ num_kv_heads,
171
scale,
172
block_tables,
173
context_lens,
@@ -202,7 +199,7 @@ def test_paged_attention(
199
query,
200
key_cache,
201
value_cache,
205
- head_mapping,
202
+ num_kv_heads,
203
scale,
204
block_tables,
205
context_lens,
vllm/model_executor/layers/attention.py
+5
-8
@@ -54,9 +54,6 @@ class PagedAttention(nn.Module):
54
55
assert self.num_heads % self.num_kv_heads == 0
56
self.num_queries_per_kv = self.num_heads // self.num_kv_heads
57
- self.head_mapping = torch.repeat_interleave(
58
- torch.arange(self.num_kv_heads, dtype=torch.int32, device="cuda"),
59
- self.num_queries_per_kv)
57
58
if self.head_size not in _SUPPORTED_HEAD_SIZES:
59
raise ValueError(f"head_size ({self.head_size}) is not supported. "
@@ -77,7 +74,7 @@ class PagedAttention(nn.Module):
74
Args:
75
query: shape = [batch_size, seq_len, num_heads * head_size]
76
key: shape = [batch_size, seq_len, num_kv_heads * head_size]
80
- value: shape = [batch_size, num_kv_heads * head_size]
77
+ value: shape = [batch_size, seq_len, num_kv_heads * head_size]
78
key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
79
block_size, x]
80
value_cache: shape = [num_blocks, num_kv_heads, head_size,
@@ -172,7 +169,7 @@ class PagedAttention(nn.Module):
169
key_cache,
170
value_cache,
171
input_metadata,
175
- self.head_mapping,
172
+ self.num_kv_heads,
173
self.scale,
174
self.alibi_slopes,
175
)
@@ -217,7 +214,7 @@ def _paged_attention(
214
key_cache: torch.Tensor,
215
value_cache: torch.Tensor,
216
input_metadata: InputMetadata,
220
- head_mapping: torch.Tensor,
217
+ num_kv_heads: int,
218
scale: float,
219
alibi_slopes: Optional[torch.Tensor],
220
) -> torch.Tensor:
@@ -244,7 +241,7 @@ def _paged_attention(
241
query,
242
key_cache,
243
value_cache,
247
- head_mapping,
244
+ num_kv_heads,
245
scale,
246
input_metadata.block_tables,
247
input_metadata.context_lens,
@@ -274,7 +271,7 @@ def _paged_attention(
271
query,
272
key_cache,
273
value_cache,
277
- head_mapping,
274
+ num_kv_heads,
275
scale,
276
input_metadata.block_tables,
277
input_metadata.context_lens,