A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Optimize MQA Kernel (#452)
Zhuohan Li committed
Jul 14, 2023 at 20:06 UTC
96853af5a830d42496aa6cfd5c670d073d6a0209
5 files changed
+84
-72
csrc/attention.cpp
+1
@@ -6,6 +6,7 @@ void single_query_cached_kv_attention(
6
torch::Tensor& query,
7
torch::Tensor& key_cache,
8
torch::Tensor& value_cache,
9
+ torch::Tensor& head_mapping,
10
float scale,
11
torch::Tensor& block_tables,
12
torch::Tensor& context_lens,
csrc/attention/attention_kernels.cu
+22
-9
@@ -74,14 +74,17 @@ template<
74
__global__ void single_query_cached_kv_attention_kernel(
75
scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
76
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
77
- const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
78
- const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
77
+ const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
78
+ const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
79
+ const int* __restrict__ head_mapping, // [num_heads]
80
const float scale,
81
const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
82
const int* __restrict__ context_lens, // [num_seqs]
83
const int max_num_blocks_per_seq,
84
const float* __restrict__ alibi_slopes, // [num_heads]
84
- const int q_stride) {
85
+ const int q_stride,
86
+ const int kv_block_stride,
87
+ const int kv_head_stride) {
88
constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
89
constexpr int NUM_TOKENS_PER_THREAD_GROUP = (BLOCK_SIZE + WARP_SIZE - 1) / WARP_SIZE;
90
constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
@@ -91,6 +94,7 @@ __global__ void single_query_cached_kv_attention_kernel(
94
95
const int head_idx = blockIdx.x;
96
const int num_heads = gridDim.x;
97
+ const int kv_head_idx = head_mapping[head_idx];
98
const int seq_idx = blockIdx.y;
99
const float alibi_slope = alibi_slopes == nullptr ? 0.f : alibi_slopes[head_idx];
100
@@ -158,8 +162,8 @@ __global__ void single_query_cached_kv_attention_kernel(
162
163
#pragma unroll
164
for (int j = 0; j < NUM_VECS_PER_THREAD; j++) {
161
- const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
162
- + head_idx * HEAD_SIZE * BLOCK_SIZE
165
+ const scalar_t* k_ptr = k_cache + physical_block_number * kv_block_stride
166
+ + kv_head_idx * kv_head_stride
167
+ physical_block_offset * x;
168
const int vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;
169
const int offset1 = (vec_idx * VEC_SIZE) / x;
@@ -246,8 +250,8 @@ __global__ void single_query_cached_kv_attention_kernel(
250
L_vec logits_vec;
251
from_float(logits_vec, *reinterpret_cast<Float_L_vec*>(logits + token_idx));
252
249
- const scalar_t* v_ptr = v_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
250
- + head_idx * HEAD_SIZE * BLOCK_SIZE;
253
+ const scalar_t* v_ptr = v_cache + physical_block_number * kv_block_stride
254
+ + kv_head_idx * kv_head_stride;
255
#pragma unroll
256
for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
257
const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
@@ -328,12 +332,15 @@ __global__ void single_query_cached_kv_attention_kernel(
332
query_ptr, \
333
key_cache_ptr, \
334
value_cache_ptr, \
335
+ head_mapping_ptr, \
336
scale, \
337
block_tables_ptr, \
338
context_lens_ptr, \
339
max_num_blocks_per_seq, \
340
alibi_slopes_ptr, \
336
- query_stride);
341
+ q_stride, \
342
+ kv_block_stride, \
343
+ kv_head_stride);
344
345
// TODO(woosuk): Tune NUM_THREADS.
346
template<
@@ -345,6 +352,7 @@ void single_query_cached_kv_attention_launcher(
352
torch::Tensor& query,
353
torch::Tensor& key_cache,
354
torch::Tensor& value_cache,
355
+ torch::Tensor& head_mapping,
356
float scale,
357
torch::Tensor& block_tables,
358
torch::Tensor& context_lens,
@@ -354,7 +362,9 @@ void single_query_cached_kv_attention_launcher(
362
int num_heads = query.size(1);
363
int head_size = query.size(2);
364
int max_num_blocks_per_seq = block_tables.size(1);
357
- int query_stride = query.stride(0);
365
+ int q_stride = query.stride(0);
366
+ int kv_block_stride = key_cache.stride(0);
367
+ int kv_head_stride = key_cache.stride(1);
368
369
int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
370
assert(head_size % thread_group_size == 0);
@@ -368,6 +378,7 @@ void single_query_cached_kv_attention_launcher(
378
T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
379
T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
380
T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
381
+ int* head_mapping_ptr = reinterpret_cast<int*>(head_mapping.data_ptr());
382
int* block_tables_ptr = block_tables.data_ptr<int>();
383
int* context_lens_ptr = context_lens.data_ptr<int>();
384
@@ -422,6 +433,7 @@ void single_query_cached_kv_attention_launcher(
433
query, \
434
key_cache, \
435
value_cache, \
436
+ head_mapping, \
437
scale, \
438
block_tables, \
439
context_lens, \
@@ -469,6 +481,7 @@ void single_query_cached_kv_attention(
481
torch::Tensor& query, // [num_seqs, num_heads, head_size]
482
torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
483
torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
484
+ torch::Tensor& head_mapping, // [num_heads]
485
float scale,
486
torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
487
torch::Tensor& context_lens, // [num_seqs]
vllm/config.py
+7
@@ -94,6 +94,13 @@ class ModelConfig:
94
return self.hf_config.hidden_size // self.hf_config.num_attention_heads
95
96
def get_num_heads(self, parallel_config: "ParallelConfig") -> int:
97
+ # For GPTBigCode:
98
+ if getattr(self.hf_config, "multi_query", False):
99
+ # Multi-query attention, only one KV head.
100
+ return 1
101
+ # For Falcon:
102
+ if getattr(self.hf_config, "n_head_kv", None) is not None:
103
+ return self.hf_config.n_head_kv
104
total_num_attention_heads = self.hf_config.num_attention_heads
105
return total_num_attention_heads // parallel_config.tensor_parallel_size
106
vllm/model_executor/layers/attention.py
+32
-11
@@ -44,12 +44,23 @@ class PagedAttention(nn.Module):
44
5. Output a flattened 1D tensor.
45
"""
46
47
- def __init__(self, num_heads: int, head_size: int, scale: float) -> None:
47
+ def __init__(self,
48
+ num_heads: int,
49
+ head_size: int,
50
+ scale: float,
51
+ num_kv_heads: Optional[int] = None) -> None:
52
super().__init__()
53
self.num_heads = num_heads
54
self.head_size = head_size
55
self.scale = float(scale)
56
self.attn_op = xops.fmha.cutlass.FwOp()
57
+ self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
58
+
59
+ assert self.num_heads % self.num_kv_heads == 0
60
+ self.num_queries_per_kv = self.num_heads // self.num_kv_heads
61
+ self.head_mapping = torch.repeat_interleave(
62
+ torch.arange(self.num_kv_heads, dtype=torch.int32, device="cuda"),
63
+ self.num_queries_per_kv)
64
65
if self.head_size not in _SUPPORTED_HEAD_SIZES:
66
raise ValueError(f"head_size ({self.head_size}) is not supported. "
@@ -76,10 +87,18 @@ class PagedAttention(nn.Module):
87
Args:
88
output: shape = [num_prompt_tokens, num_heads, head_size]
89
query: shape = [num_prompt_tokens, num_heads, head_size]
79
- key: shape = [num_prompt_tokens, num_heads, head_size]
80
- value: shape = [num_prompt_tokens, num_heads, head_size]
90
+ key: shape = [num_prompt_tokens, num_kv_heads, head_size]
91
+ value: shape = [num_prompt_tokens, num_kv_heads, head_size]
92
input_metadata: metadata for paged attention.
93
"""
94
+
95
+ if self.num_kv_heads != self.num_heads:
96
+ # Project the key and value tensors to the desired number of heads.
97
+ key = torch.repeat_interleave(key, self.num_queries_per_kv, dim=1)
98
+ value = torch.repeat_interleave(value,
99
+ self.num_queries_per_kv,
100
+ dim=1)
101
+
102
# TODO(woosuk): The unsqueeze op may incur some CPU overhead. Optimize.
103
out = xops.memory_efficient_attention_forward(
104
query.unsqueeze(0),
@@ -107,9 +126,9 @@ class PagedAttention(nn.Module):
126
Args:
127
output: shape = [num_generation_tokens, num_heads, head_size]
128
query: shape = [num_generation_tokens, num_heads, head_size]
110
- key_cache: shape = [num_blocks, num_heads, head_size/x,
129
+ key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
130
block_size, x]
112
- value_cache: shape = [num_blocks, num_heads, head_size, block_size]
131
+ value_cache: shape = [num_blocks, num_kv_heads, head_size, block_size]
132
input_metadata: metadata for paged attention.
133
"""
134
block_size = value_cache.shape[3]
@@ -118,6 +137,7 @@ class PagedAttention(nn.Module):
137
query,
138
key_cache,
139
value_cache,
140
+ self.head_mapping,
141
self.scale,
142
input_metadata.block_tables,
143
input_metadata.context_lens,
@@ -143,11 +163,12 @@ class PagedAttention(nn.Module):
163
164
Args:
165
query: shape = [num_tokens, num_heads * head_size]
146
- key: shape = [num_tokens, num_heads * head_size]
147
- value: shape = [num_tokens, num_heads * head_size]
148
- key_cache: shape = [num_blocks, num_heads, head_size/x,
166
+ key: shape = [num_tokens, num_kv_heads * head_size]
167
+ value: shape = [num_tokens, num_kv_heads * head_size]
168
+ key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
169
block_size, x]
150
- value_cache: shape = [num_blocks, num_heads, head_size, block_size]
170
+ value_cache: shape = [num_blocks, num_kv_heads, head_size,
171
+ block_size]
172
input_metadata: metadata for paged attention.
173
cache_event: event to wait for the cache operations to finish.
174
@@ -157,8 +178,8 @@ class PagedAttention(nn.Module):
178
179
# Reshape the query, key, and value tensors.
180
query = query.view(-1, self.num_heads, self.head_size)
160
- key = key.view(-1, self.num_heads, self.head_size)
161
- value = value.view(-1, self.num_heads, self.head_size)
181
+ key = key.view(-1, self.num_kv_heads, self.head_size)
182
+ value = value.view(-1, self.num_kv_heads, self.head_size)
183
184
# Pre-allocate the output tensor.
185
output = torch.empty_like(query)
vllm/model_executor/models/gpt_bigcode.py
+22
-52
@@ -26,7 +26,6 @@ from typing import Dict, List, Optional, Tuple
26
27
import torch
28
from torch import nn
29
-import numpy as np
29
from transformers import GPTBigCodeConfig
30
31
from vllm.model_executor.input_metadata import InputMetadata
@@ -55,10 +54,12 @@ class GPTBigCodeAttention(nn.Module):
54
assert total_num_heads % tensor_model_parallel_world_size == 0
55
self.num_heads = total_num_heads // tensor_model_parallel_world_size
56
self.head_dim = self.hidden_size // total_num_heads
57
+ self.num_kv_heads = 1 if config.multi_query else self.num_heads
58
+ self.kv_dim = self.num_kv_heads * self.head_dim
59
self.scale = self.head_dim**-0.5
60
61
self.c_attn = ColumnParallelLinear(self.hidden_size,
61
- 3 * self.hidden_size,
62
+ self.hidden_size + 2 * self.kv_dim,
63
bias=True,
64
gather_output=False,
65
perform_initialization=False)
@@ -69,7 +70,8 @@ class GPTBigCodeAttention(nn.Module):
70
perform_initialization=False)
71
self.attn = PagedAttention(self.num_heads,
72
self.head_dim,
72
- scale=self.scale)
73
+ scale=self.scale,
74
+ num_kv_heads=self.num_kv_heads)
75
76
def forward(
77
self,
@@ -79,7 +81,8 @@ class GPTBigCodeAttention(nn.Module):
81
cache_event: Optional[torch.cuda.Event],
82
) -> torch.Tensor:
83
qkv, _ = self.c_attn(hidden_states)
82
- q, k, v = qkv.chunk(chunks=3, dim=-1)
84
+ q, k, v = qkv.split([self.hidden_size, self.kv_dim, self.kv_dim],
85
+ dim=-1)
86
key_cache, value_cache = kv_cache
87
attn_output = self.attn(q, k, v, key_cache, value_cache,
88
input_metadata, cache_event)
@@ -263,36 +266,6 @@ class GPTBigCodeForCausalLM(nn.Module):
266
extra_rows = extra_rows.to(loaded_weight)
267
loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
268
266
- def _expand_mqa_mha(qkv_array, n_head, head_dim):
267
- """manipulates along axis=0 from MQA to MHA
268
- inputs: qkv_array.shape=((n_heads + 2) * head_dim, hidden_dim)
269
- with n_heads for q, then 1 for k, 1 for 1 v, times head dim
270
- return: qkv_array.shape=(3 * n_heads * head_dim, hidden_dim)
271
-
272
- TODO: this function is no longer needed once vllm supports MQA.
273
- """
274
- qkv_array = qkv_array.numpy()
275
-
276
- dims_q = n_head * head_dim
277
- # pylint: disable=unbalanced-tuple-unpacking
278
- q, k, v = np.split(qkv_array, (dims_q, dims_q + head_dim),
279
- axis=0)
280
- # q is fine, but k & v have not replicated shape along the first
281
- # axis as long as MQA is not nativly supported, increase memory
282
- # and replicated (head_dim, hidden_dim) to
283
- # (n_heads * head_dim, hidden_dim)
284
- if k.ndim == 2 and v.ndim == 2:
285
- replication = (n_head, 1) # weights
286
- else:
287
- replication = n_head # biases
288
- # replicate n_head times for q, v
289
- k, v = np.tile(k, replication), np.tile(v, replication)
290
- # concat q, k, v along the first axis
291
- # (n_heads * head_dim, hidden_dim)
292
- # to (3 * n_heads * head_dim, hidden_dim)
293
- qkv_array = np.concatenate((q, k, v), axis=0)
294
- return torch.from_numpy(qkv_array)
295
-
269
# For the fused QKV linear layer, manually shard the weights.
270
if "c_attn" in name:
271
# GPT-2's fused QKV has the shape of
@@ -300,30 +273,27 @@ class GPTBigCodeForCausalLM(nn.Module):
273
# When tensor parallelism is used, we shard the weights along
274
# the head dimension.
275
total_num_heads = self.config.num_attention_heads
276
+ total_num_kv_heads = (1 if self.config.multi_query else
277
+ total_num_heads)
278
hidden_size = self.config.hidden_size
279
head_size = hidden_size // total_num_heads
280
+ total_kv_size = head_size * total_num_kv_heads
281
num_heads = total_num_heads // tensor_model_parallel_world_size
282
head_start = tensor_model_parallel_rank * num_heads
283
head_end = (tensor_model_parallel_rank + 1) * num_heads
284
309
- if name.endswith(".weight"):
310
- loaded_weight = _expand_mqa_mha(loaded_weight,
311
- n_head=total_num_heads,
312
- head_dim=head_size)
313
- loaded_weight = loaded_weight.view(3, total_num_heads,
314
- head_size, hidden_size)
315
- loaded_weight = loaded_weight[:, head_start:head_end, :, :]
316
- loaded_weight = loaded_weight.reshape(-1, hidden_size)
317
- elif name.endswith(".bias"):
318
- loaded_weight = _expand_mqa_mha(loaded_weight,
319
- n_head=total_num_heads,
320
- head_dim=head_size)
321
- loaded_weight = loaded_weight.view(3, total_num_heads,
322
- head_size)
323
- loaded_weight = loaded_weight[:, head_start:head_end, :]
324
- loaded_weight = loaded_weight.reshape(-1)
325
- else:
326
- raise ValueError(f"Unexpected parameter name {name}")
285
+ wq, wk, wv = torch.split(
286
+ loaded_weight, [hidden_size, total_kv_size, total_kv_size],
287
+ dim=0)
288
+
289
+ wq = wq[head_size * head_start:head_size * head_end]
290
+ if not self.config.multi_query:
291
+ # Split the heads when using normal multi-head attention
292
+ wk = wk[head_size * head_start:head_size * head_end]
293
+ wv = wv[head_size * head_start:head_size * head_end]
294
+ # Else, keep the weights as is for multi-query attention
295
+
296
+ loaded_weight = torch.cat([wq, wk, wv], dim=0)
297
298
load_tensor_parallel_weights(param, loaded_weight, name,
299
self._column_parallel_weights,