A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[OPTIMIZATION] Optimizes the single_query_cached_kv_attention kernel (#420)
Dean Leitersdorf committed
Aug 4, 2023 at 20:57 UTC
79af7e96a0e2fc9f340d1939192122c3ae38ff17
1 file changed
+7
-4
csrc/attention/attention_kernels.cu
+7
-4
@@ -86,6 +86,8 @@ __global__ void single_query_cached_kv_attention_kernel(
86
const int kv_block_stride,
87
const int kv_head_stride) {
88
constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
89
+ constexpr int NUM_THREAD_GROUPS = NUM_THREADS / THREAD_GROUP_SIZE; // Note: This assumes THREAD_GROUP_SIZE divides NUM_THREADS
90
+ assert(NUM_THREADS % THREAD_GROUP_SIZE == 0);
91
constexpr int NUM_TOKENS_PER_THREAD_GROUP = (BLOCK_SIZE + WARP_SIZE - 1) / WARP_SIZE;
92
constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
93
const int thread_idx = threadIdx.x;
@@ -120,12 +122,13 @@ __global__ void single_query_cached_kv_attention_kernel(
122
// th vectors of the query, and so on.
123
// NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
124
const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
123
- Q_vec q_vecs[NUM_VECS_PER_THREAD];
125
+ __shared__ Q_vec q_vecs[THREAD_GROUP_SIZE][NUM_VECS_PER_THREAD];
126
#pragma unroll
125
- for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
127
+ for (int i = thread_group_idx; i < NUM_VECS_PER_THREAD; i += NUM_THREAD_GROUPS) {
128
const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
127
- q_vecs[i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
129
+ q_vecs[thread_group_offset][i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
130
}
131
+ __syncthreads(); // TODO(naed90): possible speedup if this is replaced with a memory wall right before we use q_vecs
132
133
// Memory planning.
134
extern __shared__ char shared_mem[];
@@ -173,7 +176,7 @@ __global__ void single_query_cached_kv_attention_kernel(
176
177
// Compute dot product.
178
// This includes a reduction across the threads in the same thread group.
176
- float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
179
+ float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs[thread_group_offset], k_vecs);
180
// Add the ALiBi bias if slopes are given.
181
qk += (alibi_slope != 0) ? alibi_slope * (token_idx - context_len) : 0;
182