@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 96853af5a8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Optimize MQA Kernel (#452)

Zhuohan Li committed Jul 14, 2023 at 20:06 UTC 96853af5a830d42496aa6cfd5c670d073d6a0209
5 files changed +84 -72
csrc/attention.cpp
+1
@@ -6,6 +6,7 @@ void single_query_cached_kv_attention(
6 torch::Tensor& query,
7 torch::Tensor& key_cache,
8 torch::Tensor& value_cache,
9 + torch::Tensor& head_mapping,
10 float scale,
11 torch::Tensor& block_tables,
12 torch::Tensor& context_lens,
csrc/attention/attention_kernels.cu
+22 -9
@@ -74,14 +74,17 @@ template<
74 __global__ void single_query_cached_kv_attention_kernel(
75 scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
76 const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
77 - const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
78 - const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
77 + const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
78 + const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
79 + const int* __restrict__ head_mapping, // [num_heads]
80 const float scale,
81 const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
82 const int* __restrict__ context_lens, // [num_seqs]
83 const int max_num_blocks_per_seq,
84 const float* __restrict__ alibi_slopes, // [num_heads]
84 - const int q_stride) {
85 + const int q_stride,
86 + const int kv_block_stride,
87 + const int kv_head_stride) {
88 constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
89 constexpr int NUM_TOKENS_PER_THREAD_GROUP = (BLOCK_SIZE + WARP_SIZE - 1) / WARP_SIZE;
90 constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
@@ -91,6 +94,7 @@ __global__ void single_query_cached_kv_attention_kernel(
94
95 const int head_idx = blockIdx.x;
96 const int num_heads = gridDim.x;
97 + const int kv_head_idx = head_mapping[head_idx];
98 const int seq_idx = blockIdx.y;
99 const float alibi_slope = alibi_slopes == nullptr ? 0.f : alibi_slopes[head_idx];
100
@@ -158,8 +162,8 @@ __global__ void single_query_cached_kv_attention_kernel(
162
163 #pragma unroll
164 for (int j = 0; j < NUM_VECS_PER_THREAD; j++) {
161 - const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
162 - + head_idx * HEAD_SIZE * BLOCK_SIZE
165 + const scalar_t* k_ptr = k_cache + physical_block_number * kv_block_stride
166 + + kv_head_idx * kv_head_stride
167 + physical_block_offset * x;
168 const int vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;
169 const int offset1 = (vec_idx * VEC_SIZE) / x;
@@ -246,8 +250,8 @@ __global__ void single_query_cached_kv_attention_kernel(
250 L_vec logits_vec;
251 from_float(logits_vec, *reinterpret_cast<Float_L_vec*>(logits + token_idx));
252
249 - const scalar_t* v_ptr = v_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
250 - + head_idx * HEAD_SIZE * BLOCK_SIZE;
253 + const scalar_t* v_ptr = v_cache + physical_block_number * kv_block_stride
254 + + kv_head_idx * kv_head_stride;
255 #pragma unroll
256 for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
257 const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
@@ -328,12 +332,15 @@ __global__ void single_query_cached_kv_attention_kernel(
332 query_ptr, \
333 key_cache_ptr, \
334 value_cache_ptr, \
335 + head_mapping_ptr, \
336 scale, \
337 block_tables_ptr, \
338 context_lens_ptr, \
339 max_num_blocks_per_seq, \
340 alibi_slopes_ptr, \
336 - query_stride);
341 + q_stride, \
342 + kv_block_stride, \
343 + kv_head_stride);
344
345 // TODO(woosuk): Tune NUM_THREADS.
346 template<
@@ -345,6 +352,7 @@ void single_query_cached_kv_attention_launcher(
352 torch::Tensor& query,
353 torch::Tensor& key_cache,
354 torch::Tensor& value_cache,
355 + torch::Tensor& head_mapping,
356 float scale,
357 torch::Tensor& block_tables,
358 torch::Tensor& context_lens,
@@ -354,7 +362,9 @@ void single_query_cached_kv_attention_launcher(
362 int num_heads = query.size(1);
363 int head_size = query.size(2);
364 int max_num_blocks_per_seq = block_tables.size(1);
357 - int query_stride = query.stride(0);
365 + int q_stride = query.stride(0);
366 + int kv_block_stride = key_cache.stride(0);
367 + int kv_head_stride = key_cache.stride(1);
368
369 int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
370 assert(head_size % thread_group_size == 0);
@@ -368,6 +378,7 @@ void single_query_cached_kv_attention_launcher(
378 T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
379 T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
380 T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
381 + int* head_mapping_ptr = reinterpret_cast<int*>(head_mapping.data_ptr());
382 int* block_tables_ptr = block_tables.data_ptr<int>();
383 int* context_lens_ptr = context_lens.data_ptr<int>();
384
@@ -422,6 +433,7 @@ void single_query_cached_kv_attention_launcher(
433 query, \
434 key_cache, \
435 value_cache, \
436 + head_mapping, \
437 scale, \
438 block_tables, \
439 context_lens, \
@@ -469,6 +481,7 @@ void single_query_cached_kv_attention(
481 torch::Tensor& query, // [num_seqs, num_heads, head_size]
482 torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
483 torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
484 + torch::Tensor& head_mapping, // [num_heads]
485 float scale,
486 torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
487 torch::Tensor& context_lens, // [num_seqs]
vllm/config.py
+7
@@ -94,6 +94,13 @@ class ModelConfig:
94 return self.hf_config.hidden_size // self.hf_config.num_attention_heads
95
96 def get_num_heads(self, parallel_config: "ParallelConfig") -> int:
97 + # For GPTBigCode:
98 + if getattr(self.hf_config, "multi_query", False):
99 + # Multi-query attention, only one KV head.
100 + return 1
101 + # For Falcon:
102 + if getattr(self.hf_config, "n_head_kv", None) is not None:
103 + return self.hf_config.n_head_kv
104 total_num_attention_heads = self.hf_config.num_attention_heads
105 return total_num_attention_heads // parallel_config.tensor_parallel_size
106
vllm/model_executor/layers/attention.py
+32 -11
@@ -44,12 +44,23 @@ class PagedAttention(nn.Module):
44 5. Output a flattened 1D tensor.
45 """
46
47 - def __init__(self, num_heads: int, head_size: int, scale: float) -> None:
47 + def __init__(self,
48 + num_heads: int,
49 + head_size: int,
50 + scale: float,
51 + num_kv_heads: Optional[int] = None) -> None:
52 super().__init__()
53 self.num_heads = num_heads
54 self.head_size = head_size
55 self.scale = float(scale)
56 self.attn_op = xops.fmha.cutlass.FwOp()
57 + self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
58 +
59 + assert self.num_heads % self.num_kv_heads == 0
60 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads
61 + self.head_mapping = torch.repeat_interleave(
62 + torch.arange(self.num_kv_heads, dtype=torch.int32, device="cuda"),
63 + self.num_queries_per_kv)
64
65 if self.head_size not in _SUPPORTED_HEAD_SIZES:
66 raise ValueError(f"head_size ({self.head_size}) is not supported. "
@@ -76,10 +87,18 @@ class PagedAttention(nn.Module):
87 Args:
88 output: shape = [num_prompt_tokens, num_heads, head_size]
89 query: shape = [num_prompt_tokens, num_heads, head_size]
79 - key: shape = [num_prompt_tokens, num_heads, head_size]
80 - value: shape = [num_prompt_tokens, num_heads, head_size]
90 + key: shape = [num_prompt_tokens, num_kv_heads, head_size]
91 + value: shape = [num_prompt_tokens, num_kv_heads, head_size]
92 input_metadata: metadata for paged attention.
93 """
94 +
95 + if self.num_kv_heads != self.num_heads:
96 + # Project the key and value tensors to the desired number of heads.
97 + key = torch.repeat_interleave(key, self.num_queries_per_kv, dim=1)
98 + value = torch.repeat_interleave(value,
99 + self.num_queries_per_kv,
100 + dim=1)
101 +
102 # TODO(woosuk): The unsqueeze op may incur some CPU overhead. Optimize.
103 out = xops.memory_efficient_attention_forward(
104 query.unsqueeze(0),
@@ -107,9 +126,9 @@ class PagedAttention(nn.Module):
126 Args:
127 output: shape = [num_generation_tokens, num_heads, head_size]
128 query: shape = [num_generation_tokens, num_heads, head_size]
110 - key_cache: shape = [num_blocks, num_heads, head_size/x,
129 + key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
130 block_size, x]
112 - value_cache: shape = [num_blocks, num_heads, head_size, block_size]
131 + value_cache: shape = [num_blocks, num_kv_heads, head_size, block_size]
132 input_metadata: metadata for paged attention.
133 """
134 block_size = value_cache.shape[3]
@@ -118,6 +137,7 @@ class PagedAttention(nn.Module):
137 query,
138 key_cache,
139 value_cache,
140 + self.head_mapping,
141 self.scale,
142 input_metadata.block_tables,
143 input_metadata.context_lens,
@@ -143,11 +163,12 @@ class PagedAttention(nn.Module):
163
164 Args:
165 query: shape = [num_tokens, num_heads * head_size]
146 - key: shape = [num_tokens, num_heads * head_size]
147 - value: shape = [num_tokens, num_heads * head_size]
148 - key_cache: shape = [num_blocks, num_heads, head_size/x,
166 + key: shape = [num_tokens, num_kv_heads * head_size]
167 + value: shape = [num_tokens, num_kv_heads * head_size]
168 + key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
169 block_size, x]
150 - value_cache: shape = [num_blocks, num_heads, head_size, block_size]
170 + value_cache: shape = [num_blocks, num_kv_heads, head_size,
171 + block_size]
172 input_metadata: metadata for paged attention.
173 cache_event: event to wait for the cache operations to finish.
174
@@ -157,8 +178,8 @@ class PagedAttention(nn.Module):
178
179 # Reshape the query, key, and value tensors.
180 query = query.view(-1, self.num_heads, self.head_size)
160 - key = key.view(-1, self.num_heads, self.head_size)
161 - value = value.view(-1, self.num_heads, self.head_size)
181 + key = key.view(-1, self.num_kv_heads, self.head_size)
182 + value = value.view(-1, self.num_kv_heads, self.head_size)
183
184 # Pre-allocate the output tensor.
185 output = torch.empty_like(query)
vllm/model_executor/models/gpt_bigcode.py
+22 -52
@@ -26,7 +26,6 @@ from typing import Dict, List, Optional, Tuple
26
27 import torch
28 from torch import nn
29 -import numpy as np
29 from transformers import GPTBigCodeConfig
30
31 from vllm.model_executor.input_metadata import InputMetadata
@@ -55,10 +54,12 @@ class GPTBigCodeAttention(nn.Module):
54 assert total_num_heads % tensor_model_parallel_world_size == 0
55 self.num_heads = total_num_heads // tensor_model_parallel_world_size
56 self.head_dim = self.hidden_size // total_num_heads
57 + self.num_kv_heads = 1 if config.multi_query else self.num_heads
58 + self.kv_dim = self.num_kv_heads * self.head_dim
59 self.scale = self.head_dim**-0.5
60
61 self.c_attn = ColumnParallelLinear(self.hidden_size,
61 - 3 * self.hidden_size,
62 + self.hidden_size + 2 * self.kv_dim,
63 bias=True,
64 gather_output=False,
65 perform_initialization=False)
@@ -69,7 +70,8 @@ class GPTBigCodeAttention(nn.Module):
70 perform_initialization=False)
71 self.attn = PagedAttention(self.num_heads,
72 self.head_dim,
72 - scale=self.scale)
73 + scale=self.scale,
74 + num_kv_heads=self.num_kv_heads)
75
76 def forward(
77 self,
@@ -79,7 +81,8 @@ class GPTBigCodeAttention(nn.Module):
81 cache_event: Optional[torch.cuda.Event],
82 ) -> torch.Tensor:
83 qkv, _ = self.c_attn(hidden_states)
82 - q, k, v = qkv.chunk(chunks=3, dim=-1)
84 + q, k, v = qkv.split([self.hidden_size, self.kv_dim, self.kv_dim],
85 + dim=-1)
86 key_cache, value_cache = kv_cache
87 attn_output = self.attn(q, k, v, key_cache, value_cache,
88 input_metadata, cache_event)
@@ -263,36 +266,6 @@ class GPTBigCodeForCausalLM(nn.Module):
266 extra_rows = extra_rows.to(loaded_weight)
267 loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
268
266 - def _expand_mqa_mha(qkv_array, n_head, head_dim):
267 - """manipulates along axis=0 from MQA to MHA
268 - inputs: qkv_array.shape=((n_heads + 2) * head_dim, hidden_dim)
269 - with n_heads for q, then 1 for k, 1 for 1 v, times head dim
270 - return: qkv_array.shape=(3 * n_heads * head_dim, hidden_dim)
271 -
272 - TODO: this function is no longer needed once vllm supports MQA.
273 - """
274 - qkv_array = qkv_array.numpy()
275 -
276 - dims_q = n_head * head_dim
277 - # pylint: disable=unbalanced-tuple-unpacking
278 - q, k, v = np.split(qkv_array, (dims_q, dims_q + head_dim),
279 - axis=0)
280 - # q is fine, but k & v have not replicated shape along the first
281 - # axis as long as MQA is not nativly supported, increase memory
282 - # and replicated (head_dim, hidden_dim) to
283 - # (n_heads * head_dim, hidden_dim)
284 - if k.ndim == 2 and v.ndim == 2:
285 - replication = (n_head, 1) # weights
286 - else:
287 - replication = n_head # biases
288 - # replicate n_head times for q, v
289 - k, v = np.tile(k, replication), np.tile(v, replication)
290 - # concat q, k, v along the first axis
291 - # (n_heads * head_dim, hidden_dim)
292 - # to (3 * n_heads * head_dim, hidden_dim)
293 - qkv_array = np.concatenate((q, k, v), axis=0)
294 - return torch.from_numpy(qkv_array)
295 -
269 # For the fused QKV linear layer, manually shard the weights.
270 if "c_attn" in name:
271 # GPT-2's fused QKV has the shape of
@@ -300,30 +273,27 @@ class GPTBigCodeForCausalLM(nn.Module):
273 # When tensor parallelism is used, we shard the weights along
274 # the head dimension.
275 total_num_heads = self.config.num_attention_heads
276 + total_num_kv_heads = (1 if self.config.multi_query else
277 + total_num_heads)
278 hidden_size = self.config.hidden_size
279 head_size = hidden_size // total_num_heads
280 + total_kv_size = head_size * total_num_kv_heads
281 num_heads = total_num_heads // tensor_model_parallel_world_size
282 head_start = tensor_model_parallel_rank * num_heads
283 head_end = (tensor_model_parallel_rank + 1) * num_heads
284
309 - if name.endswith(".weight"):
310 - loaded_weight = _expand_mqa_mha(loaded_weight,
311 - n_head=total_num_heads,
312 - head_dim=head_size)
313 - loaded_weight = loaded_weight.view(3, total_num_heads,
314 - head_size, hidden_size)
315 - loaded_weight = loaded_weight[:, head_start:head_end, :, :]
316 - loaded_weight = loaded_weight.reshape(-1, hidden_size)
317 - elif name.endswith(".bias"):
318 - loaded_weight = _expand_mqa_mha(loaded_weight,
319 - n_head=total_num_heads,
320 - head_dim=head_size)
321 - loaded_weight = loaded_weight.view(3, total_num_heads,
322 - head_size)
323 - loaded_weight = loaded_weight[:, head_start:head_end, :]
324 - loaded_weight = loaded_weight.reshape(-1)
325 - else:
326 - raise ValueError(f"Unexpected parameter name {name}")
285 + wq, wk, wv = torch.split(
286 + loaded_weight, [hidden_size, total_kv_size, total_kv_size],
287 + dim=0)
288 +
289 + wq = wq[head_size * head_start:head_size * head_end]
290 + if not self.config.multi_query:
291 + # Split the heads when using normal multi-head attention
292 + wk = wk[head_size * head_start:head_size * head_end]
293 + wv = wv[head_size * head_start:head_size * head_end]
294 + # Else, keep the weights as is for multi-query attention
295 +
296 + loaded_weight = torch.cat([wq, wk, wv], dim=0)
297
298 load_tensor_parallel_weights(param, loaded_weight, name,
299 self._column_parallel_weights,