@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 79af7e96a0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[OPTIMIZATION] Optimizes the single_query_cached_kv_attention kernel (#420)

Dean Leitersdorf committed Aug 4, 2023 at 20:57 UTC 79af7e96a0e2fc9f340d1939192122c3ae38ff17
1 file changed +7 -4
csrc/attention/attention_kernels.cu
+7 -4
@@ -86,6 +86,8 @@ __global__ void single_query_cached_kv_attention_kernel(
86 const int kv_block_stride,
87 const int kv_head_stride) {
88 constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
89 + constexpr int NUM_THREAD_GROUPS = NUM_THREADS / THREAD_GROUP_SIZE; // Note: This assumes THREAD_GROUP_SIZE divides NUM_THREADS
90 + assert(NUM_THREADS % THREAD_GROUP_SIZE == 0);
91 constexpr int NUM_TOKENS_PER_THREAD_GROUP = (BLOCK_SIZE + WARP_SIZE - 1) / WARP_SIZE;
92 constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
93 const int thread_idx = threadIdx.x;
@@ -120,12 +122,13 @@ __global__ void single_query_cached_kv_attention_kernel(
122 // th vectors of the query, and so on.
123 // NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
124 const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
123 - Q_vec q_vecs[NUM_VECS_PER_THREAD];
125 + __shared__ Q_vec q_vecs[THREAD_GROUP_SIZE][NUM_VECS_PER_THREAD];
126 #pragma unroll
125 - for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
127 + for (int i = thread_group_idx; i < NUM_VECS_PER_THREAD; i += NUM_THREAD_GROUPS) {
128 const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
127 - q_vecs[i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
129 + q_vecs[thread_group_offset][i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
130 }
131 + __syncthreads(); // TODO(naed90): possible speedup if this is replaced with a memory wall right before we use q_vecs
132
133 // Memory planning.
134 extern __shared__ char shared_mem[];
@@ -173,7 +176,7 @@ __global__ void single_query_cached_kv_attention_kernel(
176
177 // Compute dot product.
178 // This includes a reduction across the threads in the same thread group.
176 - float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
179 + float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs[thread_group_offset], k_vecs);
180 // Add the ALiBi bias if slopes are given.
181 qk += (alibi_slope != 0) ? alibi_slope * (token_idx - context_len) : 0;
182