@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dacaf5a400
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Replace head_mapping params with num_kv_heads to attention kernel. (#1997)

Co-authored-by: wangguoya <wangguoya@baidu.com> Co-authored-by: Yang Zhao <zhaoyangstar@foxmail.com>

wbn committed Dec 11, 2023 at 02:12 UTC dacaf5a40056c40be4a84f0aec42278335f76dae
5 files changed +26 -37
benchmarks/kernels/benchmark_paged_attention.py
+2 -6
@@ -37,10 +37,6 @@ def main(
37 query.uniform_(-scale, scale)
38
39 assert num_query_heads % num_kv_heads == 0
40 - num_queries_per_kv = num_query_heads // num_kv_heads
41 - head_mapping = torch.repeat_interleave(
42 - torch.arange(num_kv_heads, dtype=torch.int32, device="cuda"),
43 - num_queries_per_kv)
40 alibi_slopes = None
41 if use_alibi:
42 alibi_slopes = torch.randn(num_query_heads,
@@ -103,7 +99,7 @@ def main(
99 query,
100 key_cache,
101 value_cache,
106 - head_mapping,
102 + num_kv_heads,
103 scale,
104 block_tables,
105 context_lens,
@@ -120,7 +116,7 @@ def main(
116 query,
117 key_cache,
118 value_cache,
123 - head_mapping,
119 + num_kv_heads,
120 scale,
121 block_tables,
122 context_lens,
csrc/attention/attention_kernels.cu
+15 -16
@@ -89,7 +89,7 @@ __device__ void paged_attention_kernel(
89 const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
90 const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
91 const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
92 - const int* __restrict__ head_mapping, // [num_heads]
92 + const int num_kv_heads, // [num_heads]
93 const float scale,
94 const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
95 const int* __restrict__ context_lens, // [num_seqs]
@@ -132,7 +132,8 @@ __device__ void paged_attention_kernel(
132
133 const int head_idx = blockIdx.x;
134 const int num_heads = gridDim.x;
135 - const int kv_head_idx = head_mapping[head_idx];
135 + const int num_queries_per_kv = num_heads / num_kv_heads;
136 + const int kv_head_idx = head_idx / num_queries_per_kv;
137 const float alibi_slope = alibi_slopes == nullptr ? 0.f : alibi_slopes[head_idx];
138
139 // A vector type to store a part of a key or a query.
@@ -401,7 +402,7 @@ __global__ void paged_attention_v1_kernel(
402 const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
403 const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
404 const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
404 - const int* __restrict__ head_mapping, // [num_heads]
405 + const int num_kv_heads, // [num_heads]
406 const float scale,
407 const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
408 const int* __restrict__ context_lens, // [num_seqs]
@@ -412,7 +413,7 @@ __global__ void paged_attention_v1_kernel(
413 const int kv_head_stride) {
414 paged_attention_kernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>(
415 /* exp_sums */ nullptr, /* max_logits */ nullptr,
415 - out, q, k_cache, v_cache, head_mapping, scale, block_tables, context_lens,
416 + out, q, k_cache, v_cache, num_kv_heads, scale, block_tables, context_lens,
417 max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride, kv_head_stride);
418 }
419
@@ -430,7 +431,7 @@ __global__ void paged_attention_v2_kernel(
431 const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
432 const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
433 const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
433 - const int* __restrict__ head_mapping, // [num_heads]
434 + const int num_kv_heads, // [num_heads]
435 const float scale,
436 const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
437 const int* __restrict__ context_lens, // [num_seqs]
@@ -440,7 +441,7 @@ __global__ void paged_attention_v2_kernel(
441 const int kv_block_stride,
442 const int kv_head_stride) {
443 paged_attention_kernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, PARTITION_SIZE>(
443 - exp_sums, max_logits, tmp_out, q, k_cache, v_cache, head_mapping, scale,
444 + exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
445 block_tables, context_lens, max_num_blocks_per_seq, alibi_slopes,
446 q_stride, kv_block_stride, kv_head_stride);
447 }
@@ -556,7 +557,7 @@ __global__ void paged_attention_v2_reduce_kernel(
557 query_ptr, \
558 key_cache_ptr, \
559 value_cache_ptr, \
559 - head_mapping_ptr, \
560 + num_kv_heads, \
561 scale, \
562 block_tables_ptr, \
563 context_lens_ptr, \
@@ -576,7 +577,7 @@ void paged_attention_v1_launcher(
577 torch::Tensor& query,
578 torch::Tensor& key_cache,
579 torch::Tensor& value_cache,
579 - torch::Tensor& head_mapping,
580 + int num_kv_heads,
581 float scale,
582 torch::Tensor& block_tables,
583 torch::Tensor& context_lens,
@@ -602,7 +603,6 @@ void paged_attention_v1_launcher(
603 T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
604 T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
605 T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
605 - int* head_mapping_ptr = reinterpret_cast<int*>(head_mapping.data_ptr());
606 int* block_tables_ptr = block_tables.data_ptr<int>();
607 int* context_lens_ptr = context_lens.data_ptr<int>();
608
@@ -651,7 +651,7 @@ void paged_attention_v1_launcher(
651 query, \
652 key_cache, \
653 value_cache, \
654 - head_mapping, \
654 + num_kv_heads, \
655 scale, \
656 block_tables, \
657 context_lens, \
@@ -681,7 +681,7 @@ void paged_attention_v1(
681 torch::Tensor& query, // [num_seqs, num_heads, head_size]
682 torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
683 torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
684 - torch::Tensor& head_mapping, // [num_heads]
684 + int num_kv_heads, // [num_heads]
685 float scale,
686 torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
687 torch::Tensor& context_lens, // [num_seqs]
@@ -708,7 +708,7 @@ void paged_attention_v1(
708 query_ptr, \
709 key_cache_ptr, \
710 value_cache_ptr, \
711 - head_mapping_ptr, \
711 + num_kv_heads, \
712 scale, \
713 block_tables_ptr, \
714 context_lens_ptr, \
@@ -739,7 +739,7 @@ void paged_attention_v2_launcher(
739 torch::Tensor& query,
740 torch::Tensor& key_cache,
741 torch::Tensor& value_cache,
742 - torch::Tensor& head_mapping,
742 + int num_kv_heads,
743 float scale,
744 torch::Tensor& block_tables,
745 torch::Tensor& context_lens,
@@ -768,7 +768,6 @@ void paged_attention_v2_launcher(
768 T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
769 T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
770 T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
771 - int* head_mapping_ptr = reinterpret_cast<int*>(head_mapping.data_ptr());
771 int* block_tables_ptr = block_tables.data_ptr<int>();
772 int* context_lens_ptr = context_lens.data_ptr<int>();
773
@@ -823,7 +822,7 @@ void paged_attention_v2_launcher(
822 query, \
823 key_cache, \
824 value_cache, \
826 - head_mapping, \
825 + num_kv_heads, \
826 scale, \
827 block_tables, \
828 context_lens, \
@@ -856,7 +855,7 @@ void paged_attention_v2(
855 torch::Tensor& query, // [num_seqs, num_heads, head_size]
856 torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
857 torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
859 - torch::Tensor& head_mapping, // [num_heads]
858 + int num_kv_heads, // [num_heads]
859 float scale,
860 torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
861 torch::Tensor& context_lens, // [num_seqs]
csrc/ops.h
+2 -2
@@ -5,7 +5,7 @@ void paged_attention_v1(
5 torch::Tensor& query,
6 torch::Tensor& key_cache,
7 torch::Tensor& value_cache,
8 - torch::Tensor& head_mapping,
8 + int num_kv_heads,
9 float scale,
10 torch::Tensor& block_tables,
11 torch::Tensor& context_lens,
@@ -21,7 +21,7 @@ void paged_attention_v2(
21 torch::Tensor& query,
22 torch::Tensor& key_cache,
23 torch::Tensor& value_cache,
24 - torch::Tensor& head_mapping,
24 + int num_kv_heads,
25 float scale,
26 torch::Tensor& block_tables,
27 torch::Tensor& context_lens,
tests/kernels/test_attention.py
+2 -5
@@ -131,9 +131,6 @@ def test_paged_attention(
131
132 assert num_query_heads % num_kv_heads == 0
133 num_queries_per_kv = num_query_heads // num_kv_heads
134 - head_mapping = torch.repeat_interleave(
135 - torch.arange(num_kv_heads, dtype=torch.int32, device="cuda"),
136 - num_queries_per_kv)
134 alibi_slopes = None
135 if use_alibi:
136 alibi_slopes = torch.randn(num_query_heads,
@@ -170,7 +167,7 @@ def test_paged_attention(
167 query,
168 key_cache,
169 value_cache,
173 - head_mapping,
170 + num_kv_heads,
171 scale,
172 block_tables,
173 context_lens,
@@ -202,7 +199,7 @@ def test_paged_attention(
199 query,
200 key_cache,
201 value_cache,
205 - head_mapping,
202 + num_kv_heads,
203 scale,
204 block_tables,
205 context_lens,
vllm/model_executor/layers/attention.py
+5 -8
@@ -54,9 +54,6 @@ class PagedAttention(nn.Module):
54
55 assert self.num_heads % self.num_kv_heads == 0
56 self.num_queries_per_kv = self.num_heads // self.num_kv_heads
57 - self.head_mapping = torch.repeat_interleave(
58 - torch.arange(self.num_kv_heads, dtype=torch.int32, device="cuda"),
59 - self.num_queries_per_kv)
57
58 if self.head_size not in _SUPPORTED_HEAD_SIZES:
59 raise ValueError(f"head_size ({self.head_size}) is not supported. "
@@ -77,7 +74,7 @@ class PagedAttention(nn.Module):
74 Args:
75 query: shape = [batch_size, seq_len, num_heads * head_size]
76 key: shape = [batch_size, seq_len, num_kv_heads * head_size]
80 - value: shape = [batch_size, num_kv_heads * head_size]
77 + value: shape = [batch_size, seq_len, num_kv_heads * head_size]
78 key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
79 block_size, x]
80 value_cache: shape = [num_blocks, num_kv_heads, head_size,
@@ -172,7 +169,7 @@ class PagedAttention(nn.Module):
169 key_cache,
170 value_cache,
171 input_metadata,
175 - self.head_mapping,
172 + self.num_kv_heads,
173 self.scale,
174 self.alibi_slopes,
175 )
@@ -217,7 +214,7 @@ def _paged_attention(
214 key_cache: torch.Tensor,
215 value_cache: torch.Tensor,
216 input_metadata: InputMetadata,
220 - head_mapping: torch.Tensor,
217 + num_kv_heads: int,
218 scale: float,
219 alibi_slopes: Optional[torch.Tensor],
220 ) -> torch.Tensor:
@@ -244,7 +241,7 @@ def _paged_attention(
241 query,
242 key_cache,
243 value_cache,
247 - head_mapping,
244 + num_kv_heads,
245 scale,
246 input_metadata.block_tables,
247 input_metadata.context_lens,
@@ -274,7 +271,7 @@ def _paged_attention(
271 query,
272 key_cache,
273 value_cache,
277 - head_mapping,
274 + num_kv_heads,
275 scale,
276 input_metadata.block_tables,
277 input_metadata.context_lens,