@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1ef0d2efd0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Kernel][Hardware][Amd]Custom paged attention kernel for rocm (#8310)

Charlie Fu committed Sep 13, 2024 at 19:01 UTC 1ef0d2efd07f93bc7b0cfb597d8947b49e2fdaac
8 files changed +1371 -16
CMakeLists.txt
+23
@@ -324,6 +324,25 @@ define_gpu_extension_target(
324 WITH_SOABI)
325
326
327 +if(VLLM_GPU_LANG STREQUAL "HIP")
328 + #
329 + # _rocm_C extension
330 + #
331 + set(VLLM_ROCM_EXT_SRC
332 + "csrc/rocm/torch_bindings.cpp"
333 + "csrc/rocm/attention.cu")
334 +
335 + define_gpu_extension_target(
336 + _rocm_C
337 + DESTINATION vllm
338 + LANGUAGE ${VLLM_GPU_LANG}
339 + SOURCES ${VLLM_ROCM_EXT_SRC}
340 + COMPILE_FLAGS ${VLLM_GPU_FLAGS}
341 + ARCHITECTURES ${VLLM_GPU_ARCHES}
342 + USE_SABI 3
343 + WITH_SOABI)
344 +endif()
345 +
346
347 if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
348 message(STATUS "Enabling C extension.")
@@ -331,5 +350,9 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
350
351 message(STATUS "Enabling moe extension.")
352 add_dependencies(default _moe_C)
353 +endif()
354
355 +if(VLLM_GPU_LANG STREQUAL "HIP")
356 + message(STATUS "Enabling rocm extension.")
357 + add_dependencies(default _rocm_C)
358 endif()
csrc/rocm/attention.cu new
+1038
@@ -0,0 +1,1038 @@
1 +/*
2 + * Copyright (c) 2024, The vLLM team.
3 + *
4 + * Licensed under the Apache License, Version 2.0 (the "License");
5 + * you may not use this file except in compliance with the License.
6 + * You may obtain a copy of the License at
7 + *
8 + * http://www.apache.org/licenses/LICENSE-2.0
9 + *
10 + * Unless required by applicable law or agreed to in writing, software
11 + * distributed under the License is distributed on an "AS IS" BASIS,
12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 + * See the License for the specific language governing permissions and
14 + * limitations under the License.
15 + */
16 +
17 +#include <torch/all.h>
18 +#include <ATen/cuda/CUDAContext.h>
19 +#include <c10/cuda/CUDAGuard.h>
20 +#include <hip/hip_bf16.h>
21 +
22 +#include <algorithm>
23 +
24 +#if defined(__HIPCC__) && (defined(__gfx90a__) || defined(__gfx940__) || \
25 + defined(__gfx941__) || defined(__gfx942__))
26 + #define __HIP__MI300_MI250__
27 +#endif
28 +
29 +#if defined(NDEBUG)
30 + #undef NDEBUG
31 + #include <assert.h>
32 + #define UNREACHABLE_CODE assert(false);
33 + #define NDEBUG
34 +#else
35 + #define UNREACHABLE_CODE assert(false);
36 +#endif
37 +
38 +#define MAX(a, b) ((a) > (b) ? (a) : (b))
39 +#define MIN(a, b) ((a) < (b) ? (a) : (b))
40 +#define DIVIDE_ROUND_UP(a, b) (((a) + (b) - 1) / (b))
41 +#define WARP_SIZE 64
42 +
43 +#if defined(__HIP__MI300_MI250__) // TODO: Add NAVI support
44 +
45 + #define GCN_MFMA_INSTR1 __builtin_amdgcn_mfma_f32_16x16x4f32
46 + #define GCN_MFMA_INSTR __builtin_amdgcn_mfma_f32_4x4x4f16
47 +
48 +using floatx4 = __attribute__((__vector_size__(4 * sizeof(float)))) float;
49 +using float16x4 =
50 + __attribute__((__vector_size__(4 * sizeof(_Float16)))) _Float16;
51 +typedef float16x4 _Half4;
52 +typedef struct _Half8 {
53 + _Half4 xy[2];
54 +} _Half8;
55 +
56 +using bit16_t = uint16_t;
57 +using bit16x4 = __attribute__((__vector_size__(4 * sizeof(uint16_t)))) uint16_t;
58 +typedef bit16x4 _B16x4;
59 +typedef struct _B16x8 {
60 + _B16x4 xy[2];
61 +} _B16x8;
62 +
63 +////// Non temporal load stores ///////
64 +
65 +template <typename T>
66 +__device__ __forceinline__ T load(T* addr) {
67 + return addr[0];
68 +}
69 +
70 +template <typename T>
71 +__device__ __forceinline__ void store(T value, T* addr) {
72 + addr[0] = value;
73 +}
74 +
75 +template <typename T, int absz, int cbid, int blgp>
76 +__device__ __forceinline__ floatx4 gcn_mfma_instr(const _B16x4& inpA,
77 + const _B16x4& inpB,
78 + const floatx4& inpC) {
79 + if constexpr (std::is_same<T, _Float16>::value) {
80 + return __builtin_amdgcn_mfma_f32_4x4x4f16(inpA, inpB, inpC, absz, cbid,
81 + blgp);
82 + } else if constexpr (std::is_same<T, __hip_bfloat16>::value) {
83 + return __builtin_amdgcn_mfma_f32_4x4x4bf16_1k(inpA, inpB, inpC, absz, cbid,
84 + blgp);
85 + } else {
86 + static_assert(false, "unsupported 16b dtype");
87 + }
88 +}
89 +
90 +template <typename T>
91 +__device__ __forceinline__ float to_float(const T& inp) {
92 + if constexpr (std::is_same<T, _Float16>::value) {
93 + return (float)inp;
94 + } else if constexpr (std::is_same<T, __hip_bfloat16>::value) {
95 + return __bfloat162float(inp);
96 + } else {
97 + static_assert(false, "unsupported 16b dtype");
98 + }
99 +}
100 +
101 +template <typename T>
102 +__device__ __forceinline__ T from_float(const float& inp) {
103 + if constexpr (std::is_same<T, _Float16>::value) {
104 + return (_Float16)inp;
105 + } else if constexpr (std::is_same<T, __hip_bfloat16>::value) {
106 + return __float2bfloat16(inp);
107 + } else {
108 + static_assert(false, "unsupported 16b dtype");
109 + }
110 +}
111 +
112 +template <typename T>
113 +__device__ __forceinline__ _B16x4 from_floatx4(const floatx4& inp) {
114 + union tmpcvt {
115 + uint16_t u;
116 + _Float16 f;
117 + __hip_bfloat16 b;
118 + } t16;
119 + _B16x4 ret;
120 + if constexpr (std::is_same<T, _Float16>::value) {
121 + #pragma unroll
122 + for (int i = 0; i < 4; i++) {
123 + t16.f = (_Float16)inp[i];
124 + ret[i] = t16.u;
125 + }
126 + return ret;
127 + } else if constexpr (std::is_same<T, __hip_bfloat16>::value) {
128 + #pragma unroll
129 + for (int i = 0; i < 4; i++) {
130 + t16.b = __float2bfloat16(inp[i]);
131 + ret[i] = t16.u;
132 + }
133 + return ret;
134 + } else {
135 + static_assert(false, "unsupported 16b dtype");
136 + }
137 +}
138 +
139 +template <typename T>
140 +__device__ __forceinline__ _B16x4 addx4(const _B16x4& inp1,
141 + const _B16x4& inp2) {
142 + union tmpcvt {
143 + uint16_t u;
144 + _Float16 f;
145 + __hip_bfloat16 b;
146 + } t1, t2, res;
147 + _B16x4 ret;
148 + if constexpr (std::is_same<T, _Float16>::value) {
149 + #pragma unroll
150 + for (int i = 0; i < 4; i++) {
151 + t1.u = inp1[i];
152 + t2.u = inp2[i];
153 + res.f = t1.f + t2.f;
154 + ret[i] = res.u;
155 + }
156 + return ret;
157 + } else if constexpr (std::is_same<T, __hip_bfloat16>::value) {
158 + #pragma unroll
159 + for (int i = 0; i < 4; i++) {
160 + t1.u = inp1[i];
161 + t2.u = inp2[i];
162 + res.b = t1.b + t2.b;
163 + ret[i] = res.u;
164 + }
165 + return ret;
166 + } else {
167 + static_assert(false, "unsupported 16b dtype");
168 + }
169 +}
170 +
171 +///////////////////////////////////////
172 +
173 +// grid (num_seqs, num_partitions,num_heads/gqa_ratio)
174 +// block (partition size)
175 +template <typename scalar_t, int BLOCK_SIZE, int HEAD_SIZE, int NUM_THREADS,
176 + int GQA_RATIO>
177 +__global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
178 + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
179 + const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
180 + // head_size/x, block_size, x]
181 + const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
182 + // head_size, block_size]
183 + const int num_kv_heads, const float scale,
184 + const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
185 + const int* __restrict__ context_lens, // [num_seqs]
186 + const int max_num_blocks_per_seq,
187 + const float* __restrict__ alibi_slopes, // [num_heads]
188 + const int q_stride, const int kv_block_stride, const int kv_head_stride,
189 + float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
190 + float* __restrict__ max_logits, // [num_seqs, num_heads,
191 + // max_num_partitions]
192 + scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions,
193 + // head_size]
194 + scalar_t* __restrict__ final_out, // [num_seqs, num_heads, head_size]
195 + #if 0
196 + scalar_t* __restrict__ qk_out, // [num_heads, num_seqs, max_ctx_blocks,block_size]
197 + #endif
198 + int max_ctx_blocks) {
199 + constexpr int NWARPS = NUM_THREADS / WARP_SIZE;
200 + const int warpid = threadIdx.x / WARP_SIZE;
201 + const int laneid = threadIdx.x % WARP_SIZE;
202 + const int lane4id = laneid % 4;
203 +
204 + const int seq_idx = blockIdx.x;
205 + const int partition_idx = blockIdx.y;
206 + const int partition_size = blockDim.x;
207 + const int max_num_partitions = gridDim.y;
208 +
209 + const int context_len = context_lens[seq_idx];
210 + const int partition_start_token_idx = partition_idx * partition_size;
211 + // exit if partition is out of context for seq
212 + if (partition_start_token_idx >= context_len) {
213 + return;
214 + }
215 + constexpr int QHLOOP =
216 + DIVIDE_ROUND_UP(GQA_RATIO, 4); // each 4 lanes fetch 4 different qheads,
217 + // total qheads =8, so qhloop is 2
218 + constexpr int GQA_RATIO4 = 4 * QHLOOP;
219 + __shared__ float shared_qk_max[NWARPS][GQA_RATIO4 + 1];
220 + __shared__ float shared_exp_sum[NWARPS][GQA_RATIO4 + 1];
221 + _B16x8 Qlocal[QHLOOP];
222 + constexpr int x = 16 / sizeof(scalar_t);
223 + constexpr int KHELOOP = HEAD_SIZE / x;
224 + _B16x8 Klocal[KHELOOP];
225 + constexpr int VHELOOP =
226 + HEAD_SIZE /
227 + WARP_SIZE; // v head_size dimension is distributed across lanes
228 + constexpr int VTLOOP = 8; // 16 separate 4xtokens across warp -> 16/2
229 + // 8xtokens
230 + _B16x8 Vlocal[VHELOOP][VTLOOP];
231 + floatx4 dout[QHLOOP];
232 + float qk_max[QHLOOP];
233 + #pragma unroll
234 + for (int h = 0; h < QHLOOP; h++) {
235 + dout[h] = {0};
236 + qk_max[h] = -FLT_MAX;
237 + }
238 +
239 + const int wg_start_head_idx = blockIdx.z * GQA_RATIO;
240 + const int wg_start_kv_head_idx = blockIdx.z;
241 +
242 + const int warp_start_token_idx =
243 + partition_start_token_idx + warpid * WARP_SIZE;
244 +
245 + if (warp_start_token_idx >= context_len) { // warp out of context
246 + #pragma unroll
247 + for (int h = 0; h < GQA_RATIO4; h++) {
248 + shared_qk_max[warpid][h] = -FLT_MAX;
249 + shared_exp_sum[warpid][h] = 0.0f;
250 + }
251 + } else { // warp within context
252 +
253 + const int num_context_blocks = DIVIDE_ROUND_UP(context_len, BLOCK_SIZE);
254 + const int last_ctx_block = num_context_blocks - 1;
255 +
256 + const int* block_table = block_tables + seq_idx * max_num_blocks_per_seq;
257 +
258 + const int local_token_idx = threadIdx.x;
259 + const int global_token_idx = partition_start_token_idx + local_token_idx;
260 +
261 + const int block_idx = (global_token_idx < context_len)
262 + ? global_token_idx / BLOCK_SIZE
263 + : last_ctx_block;
264 + // fetch block number for q and k
265 + // int32 physical_block_number leads to overflow when multiplied with
266 + // kv_block_stride
267 + const int64_t physical_block_number =
268 + static_cast<int64_t>(block_table[block_idx]);
269 +
270 + // fetch vphysical block numbers up front
271 + constexpr int VBLOCKS = 8 * VTLOOP / BLOCK_SIZE;
272 + int vphysical_blocks[VBLOCKS];
273 +
274 + const int warp_start_block_idx = warp_start_token_idx / BLOCK_SIZE;
275 + #pragma unroll
276 + for (int b = 0; b < VBLOCKS; b++) {
277 + const int vblock_idx = warp_start_block_idx + b;
278 + const int vblock_idx_ctx =
279 + (vblock_idx <= last_ctx_block) ? vblock_idx : last_ctx_block;
280 + vphysical_blocks[b] = block_table[vblock_idx_ctx];
281 + }
282 + // each 4 lanes fetch 8 helems, so warp fetches 8*16 = 128 helems
283 + const scalar_t* q_ptr =
284 + q + seq_idx * q_stride + wg_start_head_idx * HEAD_SIZE;
285 + const _B16x8* q_ptrh8 = reinterpret_cast<const _B16x8*>(q_ptr);
286 + const int qhead_elemh8 = laneid / 4;
287 + #pragma unroll
288 + for (int h = 0; h < QHLOOP - 1; h++) {
289 + const int qhead_idx = h * 4 + lane4id;
290 + Qlocal[h] = q_ptrh8[qhead_idx * HEAD_SIZE / 8 + qhead_elemh8];
291 + }
292 + const int final_qhead_idx = 4 * (QHLOOP - 1) + lane4id;
293 + if (final_qhead_idx < GQA_RATIO) {
294 + Qlocal[QHLOOP - 1] =
295 + q_ptrh8[final_qhead_idx * HEAD_SIZE / 8 + qhead_elemh8];
296 + } else {
297 + Qlocal[QHLOOP - 1].xy[0] = {0};
298 + Qlocal[QHLOOP - 1].xy[1] = {0};
299 + }
300 +
301 + const scalar_t* k_ptr = k_cache + physical_block_number * kv_block_stride +
302 + wg_start_kv_head_idx * kv_head_stride;
303 +
304 + const int physical_block_offset =
305 + local_token_idx % BLOCK_SIZE; // since x=half8, physical_block_offset
306 + // is already cast as _H8
307 +
308 + const _B16x8* k_ptrh8 = reinterpret_cast<const _B16x8*>(k_ptr);
309 + #pragma unroll
310 + for (int d = 0; d < KHELOOP; d++) {
311 + Klocal[d] = k_ptrh8[d * BLOCK_SIZE + physical_block_offset];
312 + }
313 +
314 + float alibi_slope[QHLOOP];
315 + if (alibi_slopes != nullptr) {
316 + #pragma unroll
317 + for (int h = 0; h < QHLOOP; h++) {
318 + const int qhead_idx = h * 4 + lane4id;
319 + alibi_slope[h] = (qhead_idx < GQA_RATIO)
320 + ? alibi_slopes[wg_start_head_idx + qhead_idx]
321 + : 0.f;
322 + }
323 + }
324 +
325 + const scalar_t* v_ptr = v_cache + wg_start_kv_head_idx * kv_head_stride;
326 + const _B16x8* v_ptrh8 = reinterpret_cast<const _B16x8*>(v_ptr);
327 + // iterate over each v block
328 + #pragma unroll
329 + for (int b = 0; b < VBLOCKS; b++) {
330 + // int32 physical_block_number leads to overflow when multiplied with
331 + // kv_block_stride
332 + const int64_t vphysical_block_number =
333 + static_cast<int64_t>(vphysical_blocks[b]);
334 + const _B16x8* v_ptrh8b =
335 + v_ptrh8 + (vphysical_block_number * kv_block_stride) / 8;
336 + // iterate over each head elem (within head_size)
337 + #pragma unroll
338 + for (int h = 0; h < VHELOOP; h++) {
339 + const int head_size_elem = h * WARP_SIZE + laneid;
340 + const _B16x8* v_ptrh8be = v_ptrh8b + head_size_elem * BLOCK_SIZE / 8;
341 + // iterate over all velems within block
342 + #pragma unroll
343 + for (int d = 0; d < BLOCK_SIZE / 8; d++) {
344 + Vlocal[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
345 + }
346 + }
347 + }
348 +
349 + #pragma unroll
350 + for (int h = 0; h < QHLOOP; h++) {
351 + dout[h] = gcn_mfma_instr<scalar_t, 4, 0, 0>(Qlocal[h].xy[0],
352 + Klocal[0].xy[0], dout[h]);
353 + dout[h] = gcn_mfma_instr<scalar_t, 4, 0, 0>(Qlocal[h].xy[1],
354 + Klocal[0].xy[1], dout[h]);
355 + dout[h] = gcn_mfma_instr<scalar_t, 4, 1, 0>(Qlocal[h].xy[0],
356 + Klocal[1].xy[0], dout[h]);
357 + dout[h] = gcn_mfma_instr<scalar_t, 4, 1, 0>(Qlocal[h].xy[1],
358 + Klocal[1].xy[1], dout[h]);
359 + dout[h] = gcn_mfma_instr<scalar_t, 4, 2, 0>(Qlocal[h].xy[0],
360 + Klocal[2].xy[0], dout[h]);
361 + dout[h] = gcn_mfma_instr<scalar_t, 4, 2, 0>(Qlocal[h].xy[1],
362 + Klocal[2].xy[1], dout[h]);
363 + dout[h] = gcn_mfma_instr<scalar_t, 4, 3, 0>(Qlocal[h].xy[0],
364 + Klocal[3].xy[0], dout[h]);
365 + dout[h] = gcn_mfma_instr<scalar_t, 4, 3, 0>(Qlocal[h].xy[1],
366 + Klocal[3].xy[1], dout[h]);
367 + dout[h] = gcn_mfma_instr<scalar_t, 4, 4, 0>(Qlocal[h].xy[0],
368 + Klocal[4].xy[0], dout[h]);
369 + dout[h] = gcn_mfma_instr<scalar_t, 4, 4, 0>(Qlocal[h].xy[1],
370 + Klocal[4].xy[1], dout[h]);
371 + dout[h] = gcn_mfma_instr<scalar_t, 4, 5, 0>(Qlocal[h].xy[0],
372 + Klocal[5].xy[0], dout[h]);
373 + dout[h] = gcn_mfma_instr<scalar_t, 4, 5, 0>(Qlocal[h].xy[1],
374 + Klocal[5].xy[1], dout[h]);
375 + dout[h] = gcn_mfma_instr<scalar_t, 4, 6, 0>(Qlocal[h].xy[0],
376 + Klocal[6].xy[0], dout[h]);
377 + dout[h] = gcn_mfma_instr<scalar_t, 4, 6, 0>(Qlocal[h].xy[1],
378 + Klocal[6].xy[1], dout[h]);
379 + dout[h] = gcn_mfma_instr<scalar_t, 4, 7, 0>(Qlocal[h].xy[0],
380 + Klocal[7].xy[0], dout[h]);
381 + dout[h] = gcn_mfma_instr<scalar_t, 4, 7, 0>(Qlocal[h].xy[1],
382 + Klocal[7].xy[1], dout[h]);
383 + if constexpr (KHELOOP > 8) {
384 + dout[h] = gcn_mfma_instr<scalar_t, 4, 8, 0>(Qlocal[h].xy[0],
385 + Klocal[8].xy[0], dout[h]);
386 + dout[h] = gcn_mfma_instr<scalar_t, 4, 8, 0>(Qlocal[h].xy[1],
387 + Klocal[8].xy[1], dout[h]);
388 + dout[h] = gcn_mfma_instr<scalar_t, 4, 9, 0>(Qlocal[h].xy[0],
389 + Klocal[9].xy[0], dout[h]);
390 + dout[h] = gcn_mfma_instr<scalar_t, 4, 9, 0>(Qlocal[h].xy[1],
391 + Klocal[9].xy[1], dout[h]);
392 + dout[h] = gcn_mfma_instr<scalar_t, 4, 10, 0>(Qlocal[h].xy[0],
393 + Klocal[10].xy[0], dout[h]);
394 + dout[h] = gcn_mfma_instr<scalar_t, 4, 10, 0>(Qlocal[h].xy[1],
395 + Klocal[10].xy[1], dout[h]);
396 + dout[h] = gcn_mfma_instr<scalar_t, 4, 11, 0>(Qlocal[h].xy[0],
397 + Klocal[11].xy[0], dout[h]);
398 + dout[h] = gcn_mfma_instr<scalar_t, 4, 11, 0>(Qlocal[h].xy[1],
399 + Klocal[11].xy[1], dout[h]);
400 + dout[h] = gcn_mfma_instr<scalar_t, 4, 12, 0>(Qlocal[h].xy[0],
401 + Klocal[12].xy[0], dout[h]);
402 + dout[h] = gcn_mfma_instr<scalar_t, 4, 12, 0>(Qlocal[h].xy[1],
403 + Klocal[12].xy[1], dout[h]);
404 + dout[h] = gcn_mfma_instr<scalar_t, 4, 13, 0>(Qlocal[h].xy[0],
405 + Klocal[13].xy[0], dout[h]);
406 + dout[h] = gcn_mfma_instr<scalar_t, 4, 13, 0>(Qlocal[h].xy[1],
407 + Klocal[13].xy[1], dout[h]);
408 + dout[h] = gcn_mfma_instr<scalar_t, 4, 14, 0>(Qlocal[h].xy[0],
409 + Klocal[14].xy[0], dout[h]);
410 + dout[h] = gcn_mfma_instr<scalar_t, 4, 14, 0>(Qlocal[h].xy[1],
411 + Klocal[14].xy[1], dout[h]);
412 + dout[h] = gcn_mfma_instr<scalar_t, 4, 15, 0>(Qlocal[h].xy[0],
413 + Klocal[15].xy[0], dout[h]);
414 + dout[h] = gcn_mfma_instr<scalar_t, 4, 15, 0>(Qlocal[h].xy[1],
415 + Klocal[15].xy[1], dout[h]);
416 + } // KHELOOP>8
417 + dout[h] *= scale;
418 + }
419 + // transpose dout so that 4 token ids are in each lane, and 4 heads are across
420 + // 4 lanes
421 + #pragma unroll
422 + for (int h = 0; h < QHLOOP; h++) {
423 + floatx4 tmp = {0};
424 + #pragma unroll
425 + for (int i = 0; i < 4; i++) {
426 + const float B = (lane4id == i) ? 1.0f : 0.0f;
427 + // const float A = (global_token_idx < context_len) ? dout[h][i] : 0.0f;
428 + tmp = __builtin_amdgcn_mfma_f32_4x4x1f32(dout[h][i], B, tmp, 0, 0, 0);
429 + // tmp = __builtin_amdgcn_mfma_f32_4x4x1f32(A, B, tmp, 0, 0, 0);
430 + }
431 + dout[h] = tmp;
432 + }
433 +
434 + const int lane4_token_idx = 4 * (global_token_idx >> 2);
435 + const int alibi_offset = lane4_token_idx - context_len + 1;
436 + if (alibi_slopes != nullptr) {
437 + #pragma unroll
438 + for (int h = 0; h < QHLOOP; h++) {
439 + #pragma unroll
440 + for (int i = 0; i < 4; i++) {
441 + dout[h][i] += alibi_slope[h] * (alibi_offset + i);
442 + }
443 + }
444 + }
445 +
446 + #pragma unroll
447 + for (int h = 0; h < QHLOOP; h++) {
448 + qk_max[h] = -FLT_MAX;
449 + #pragma unroll
450 + for (int i = 0; i < 4; i++) {
451 + qk_max[h] = (lane4_token_idx + i < context_len)
452 + ? fmaxf(qk_max[h], dout[h][i])
453 + : qk_max[h];
454 + }
455 + #pragma unroll
456 + for (int mask = WARP_SIZE / 2; mask >= 4; mask /= 2) {
457 + qk_max[h] = fmaxf(qk_max[h], __shfl_xor(qk_max[h], mask));
458 + }
459 + }
460 +
461 + float exp_sum[QHLOOP];
462 + #pragma unroll
463 + for (int h = 0; h < QHLOOP; h++) {
464 + exp_sum[h] = 0.0f;
465 + #pragma unroll
466 + for (int i = 0; i < 4; i++) {
467 + dout[h][i] = (lane4_token_idx + i < context_len)
468 + ? __expf(dout[h][i] - qk_max[h])
469 + : 0.0f;
470 + exp_sum[h] += dout[h][i];
471 + }
472 + #pragma unroll
473 + for (int mask = WARP_SIZE / 2; mask >= 4; mask /= 2) {
474 + exp_sum[h] += __shfl_xor(exp_sum[h], mask);
475 + }
476 + }
477 +
478 + #pragma unroll
479 + for (int h = 0; h < QHLOOP; h++) {
480 + const int head_idx = 4 * h + lane4id;
481 + shared_qk_max[warpid][head_idx] = qk_max[h];
482 + shared_exp_sum[warpid][head_idx] = exp_sum[h];
483 + }
484 + } // warp within context
485 +
486 + __syncthreads();
487 +
488 + const int num_heads = gridDim.z * GQA_RATIO;
489 + float* max_logits_ptr =
490 + max_logits + seq_idx * num_heads * max_num_partitions + partition_idx;
491 + float* exp_sums_ptr =
492 + exp_sums + seq_idx * num_heads * max_num_partitions + partition_idx;
493 + #pragma unroll
494 + for (int h = 0; h < QHLOOP; h++) {
495 + float global_qk_max = -FLT_MAX;
496 + float warp_qk_max[NWARPS];
497 + const int head_idx = 4 * h + lane4id;
498 + #pragma unroll
499 + for (int w = 0; w < NWARPS; w++) {
500 + warp_qk_max[w] = shared_qk_max[w][head_idx];
501 + global_qk_max = fmaxf(global_qk_max, warp_qk_max[w]);
502 + }
503 + float global_exp_sum = 0.0f;
504 + #pragma unroll
505 + for (int w = 0; w < NWARPS; w++) {
506 + global_exp_sum +=
507 + shared_exp_sum[w][head_idx] * __expf(warp_qk_max[w] - global_qk_max);
508 + }
509 + if (head_idx < GQA_RATIO) {
510 + max_logits_ptr[(wg_start_head_idx + head_idx) * max_num_partitions] =
511 + global_qk_max;
512 + exp_sums_ptr[(wg_start_head_idx + head_idx) * max_num_partitions] =
513 + global_exp_sum;
514 + }
515 + const float global_inv_sum_scale = __fdividef(1.f, global_exp_sum + 1e-6f) *
516 + __expf(qk_max[h] - global_qk_max);
517 + dout[h] *= global_inv_sum_scale;
518 + }
519 + // logits[h] -> every 4 lanes hold 4 heads, each lane holds 4 tokens, there
520 + // are 4x16 tokens across warp
521 + _B16x4 logits[QHLOOP];
522 + #pragma unroll
523 + for (int h = 0; h < QHLOOP; h++) {
524 + logits[h] = from_floatx4<scalar_t>(dout[h]);
525 + }
526 +
527 + __shared__ _B16x4 vout_shared[QHLOOP][VHELOOP][WARP_SIZE][NWARPS + 1];
528 +
529 + if (warp_start_token_idx >= context_len) { // warp out of context
530 + #pragma unroll
531 + for (int qh = 0; qh < QHLOOP; qh++) {
532 + #pragma unroll
533 + for (int vh = 0; vh < VHELOOP; vh++) {
534 + vout_shared[qh][vh][laneid][warpid] = {0};
535 + }
536 + }
537 + } else { // warp in context
538 + // iterate across heads
539 + #pragma unroll
540 + for (int qh = 0; qh < QHLOOP; qh++) {
541 + // iterate over each v head elem (within head_size)
542 + #pragma unroll
543 + for (int vh = 0; vh < VHELOOP; vh++) {
544 + floatx4 acc = {0};
545 + // iterate over tokens
546 + acc = gcn_mfma_instr<scalar_t, 4, 0, 0>(logits[qh], Vlocal[vh][0].xy[0],
547 + acc);
548 + acc = gcn_mfma_instr<scalar_t, 4, 1, 0>(logits[qh], Vlocal[vh][0].xy[1],
549 + acc);
550 + acc = gcn_mfma_instr<scalar_t, 4, 2, 0>(logits[qh], Vlocal[vh][1].xy[0],
551 + acc);
552 + acc = gcn_mfma_instr<scalar_t, 4, 3, 0>(logits[qh], Vlocal[vh][1].xy[1],
553 + acc);
554 + acc = gcn_mfma_instr<scalar_t, 4, 4, 0>(logits[qh], Vlocal[vh][2].xy[0],
555 + acc);
556 + acc = gcn_mfma_instr<scalar_t, 4, 5, 0>(logits[qh], Vlocal[vh][2].xy[1],
557 + acc);
558 + acc = gcn_mfma_instr<scalar_t, 4, 6, 0>(logits[qh], Vlocal[vh][3].xy[0],
559 + acc);
560 + acc = gcn_mfma_instr<scalar_t, 4, 7, 0>(logits[qh], Vlocal[vh][3].xy[1],
561 + acc);
562 + acc = gcn_mfma_instr<scalar_t, 4, 8, 0>(logits[qh], Vlocal[vh][4].xy[0],
563 + acc);
564 + acc = gcn_mfma_instr<scalar_t, 4, 9, 0>(logits[qh], Vlocal[vh][4].xy[1],
565 + acc);
566 + acc = gcn_mfma_instr<scalar_t, 4, 10, 0>(logits[qh],
567 + Vlocal[vh][5].xy[0], acc);
568 + acc = gcn_mfma_instr<scalar_t, 4, 11, 0>(logits[qh],
569 + Vlocal[vh][5].xy[1], acc);
570 + acc = gcn_mfma_instr<scalar_t, 4, 12, 0>(logits[qh],
571 + Vlocal[vh][6].xy[0], acc);
572 + acc = gcn_mfma_instr<scalar_t, 4, 13, 0>(logits[qh],
573 + Vlocal[vh][6].xy[1], acc);
574 + acc = gcn_mfma_instr<scalar_t, 4, 14, 0>(logits[qh],
575 + Vlocal[vh][7].xy[0], acc);
576 + acc = gcn_mfma_instr<scalar_t, 4, 15, 0>(logits[qh],
577 + Vlocal[vh][7].xy[1], acc);
578 + vout_shared[qh][vh][laneid][warpid] = from_floatx4<scalar_t>(acc);
579 + }
580 + }
581 + } // warp in context
582 +
583 + __syncthreads();
584 +
585 + if (warpid == 0) {
586 + _B16x4 vout[QHLOOP][VHELOOP];
587 + // iterate across heads
588 + scalar_t* out_ptr;
589 + int out_num_partitions;
590 + if (context_len > partition_size) {
591 + out_num_partitions = max_num_partitions;
592 + out_ptr = out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE +
593 + partition_idx * HEAD_SIZE;
594 + } else {
595 + out_num_partitions = 1;
596 + out_ptr = final_out + seq_idx * num_heads * HEAD_SIZE;
597 + }
598 + #pragma unroll
599 + for (int qh = 0; qh < QHLOOP; qh++) {
600 + // iterate over each v head elem (within head_size)
601 + #pragma unroll
602 + for (int vh = 0; vh < VHELOOP; vh++) {
603 + vout[qh][vh] = {0};
604 + #pragma unroll
605 + for (int w = 0; w < NWARPS; w++) {
606 + vout[qh][vh] =
607 + addx4<scalar_t>(vout[qh][vh], vout_shared[qh][vh][laneid][w]);
608 + }
609 + const int head_size_elem = vh * WARP_SIZE + laneid;
610 + bit16_t* out_ptr_b16 = reinterpret_cast<bit16_t*>(out_ptr);
611 + #pragma unroll
612 + for (int i = 0; i < 4; i++) {
613 + const int head_idx = 4 * qh + i;
614 + if (head_idx < GQA_RATIO) {
615 + out_ptr_b16[(wg_start_head_idx + head_idx) * out_num_partitions *
616 + HEAD_SIZE +
617 + head_size_elem] = vout[qh][vh][i];
618 + }
619 + }
620 + }
621 + }
622 + }
623 +}
624 +
625 +// Grid: (num_heads, num_seqs).
626 +template <typename scalar_t, int HEAD_SIZE, int NUM_THREADS,
627 + int PARTITION_SIZE>
628 +__global__
629 +__launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
630 + scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
631 + const float* __restrict__ exp_sums, // [num_seqs, num_heads,
632 + // max_num_partitions]
633 + const float* __restrict__ max_logits, // [num_seqs, num_heads,
634 + // max_num_partitions]
635 + const scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads,
636 + // max_num_partitions, head_size]
637 + const int* __restrict__ context_lens, // [num_seqs]
638 + const int max_num_partitions) {
639 + const int num_heads = gridDim.x;
640 + const int head_idx = blockIdx.x;
641 + const int seq_idx = blockIdx.y;
642 + const int context_len = context_lens[seq_idx];
643 + const int num_partitions = DIVIDE_ROUND_UP(context_len, PARTITION_SIZE);
644 + if (num_partitions == 1) {
645 + // if num_partitions==1, main kernel will write to out directly, no work in
646 + // reduction kernel
647 + return;
648 + }
649 +
650 + constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
651 + const int warpid = threadIdx.x / WARP_SIZE;
652 + const int laneid = threadIdx.x % WARP_SIZE;
653 +
654 + __shared__ float shared_global_exp_sum;
655 + __shared__ float shared_exp_sums[2 * WARP_SIZE];
656 +
657 + if (warpid == 0) {
658 + const float* max_logits_ptr = max_logits +
659 + seq_idx * num_heads * max_num_partitions +
660 + head_idx * max_num_partitions;
661 +
662 + // valid partition is the last valid partition in case threadid > num
663 + // partitions
664 + const int valid_partition =
665 + (threadIdx.x < num_partitions) ? threadIdx.x : num_partitions - 1;
666 + const int valid_partition2 = (WARP_SIZE + threadIdx.x < num_partitions)
667 + ? WARP_SIZE + threadIdx.x
668 + : num_partitions - 1;
669 + float reg_max_logit = max_logits_ptr[valid_partition];
670 + float reg_max_logit2 = max_logits_ptr[valid_partition2];
671 + float max_logit = fmaxf(reg_max_logit, reg_max_logit2);
672 +
673 + #pragma unroll
674 + for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
675 + max_logit = fmaxf(max_logit, __shfl_xor(max_logit, mask));
676 + }
677 +
678 + const float* exp_sums_ptr = exp_sums +
679 + seq_idx * num_heads * max_num_partitions +
680 + head_idx * max_num_partitions;
681 +
682 + float global_exp_sum = 0.0f;
683 + float rescaled_exp_sum = exp_sums_ptr[valid_partition];
684 + float rescaled_exp_sum2 = exp_sums_ptr[valid_partition2];
685 + rescaled_exp_sum *=
686 + (threadIdx.x < num_partitions) ? expf(reg_max_logit - max_logit) : 0.0f;
687 + rescaled_exp_sum2 *= (threadIdx.x + WARP_SIZE < num_partitions)
688 + ? expf(reg_max_logit2 - max_logit)
689 + : 0.0f;
690 + global_exp_sum += rescaled_exp_sum + rescaled_exp_sum2;
691 + shared_exp_sums[threadIdx.x] = rescaled_exp_sum;
692 + shared_exp_sums[threadIdx.x + WARP_SIZE] = rescaled_exp_sum2;
693 +
694 + #pragma unroll
695 + for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
696 + global_exp_sum += __shfl_xor(global_exp_sum, mask);
697 + }
698 + if (threadIdx.x == 0) {
699 + shared_global_exp_sum = global_exp_sum;
700 + }
701 + } // warpid == 0
702 + const scalar_t* tmp_out_ptr =
703 + tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE +
704 + head_idx * max_num_partitions * HEAD_SIZE + threadIdx.x;
705 + constexpr int MAX_NPAR = 64;
706 + scalar_t tmps[MAX_NPAR];
707 + const float dzero = 0.0f;
708 + #pragma unroll
709 + for (int j = 0; j < MAX_NPAR; j++) {
710 + tmps[j] = from_float<scalar_t>(dzero);
711 + }
712 + const int last_partition_offset = (num_partitions - 1) * HEAD_SIZE;
713 + const int num_partition_offset = (num_partitions)*HEAD_SIZE;
714 + int idx = 0;
715 +
716 + constexpr int JCHUNK = 16;
717 +
718 + #pragma unroll
719 + for (int j = 0; j < JCHUNK * HEAD_SIZE; j += HEAD_SIZE) {
720 + // lastj is last valid partition
721 + const int lastj_offset =
722 + (j < num_partition_offset) ? j : last_partition_offset;
723 + tmps[idx] = tmp_out_ptr[lastj_offset];
724 + idx++;
725 + }
726 + __syncthreads();
727 +
728 + if (num_partitions > JCHUNK) {
729 + #pragma unroll
730 + for (int j = JCHUNK * HEAD_SIZE; j < 2 * JCHUNK * HEAD_SIZE;
731 + j += HEAD_SIZE) {
732 + const int lastj_offset =
733 + (j < num_partition_offset) ? j : last_partition_offset;
734 + tmps[idx] = tmp_out_ptr[lastj_offset];
735 + idx++;
736 + }
737 +
738 + if (num_partitions > 2 * JCHUNK) {
739 + #pragma unroll
740 + for (int j = 2 * JCHUNK * HEAD_SIZE; j < MAX_NPAR * HEAD_SIZE;
741 + j += HEAD_SIZE) {
742 + const int lastj_offset =
743 + (j < num_partition_offset) ? j : last_partition_offset;
744 + tmps[idx] = tmp_out_ptr[lastj_offset];
745 + idx++;
746 + }
747 + }
748 + } // num_partitions > JCHUNK
749 +
750 + // Aggregate tmp_out to out.
751 + float acc = 0.0f;
752 + #pragma unroll
753 + for (int j = 0; j < JCHUNK; j++) {
754 + acc += to_float<scalar_t>(tmps[j]) * shared_exp_sums[j];
755 + }
756 + if (num_partitions > JCHUNK) {
757 + #pragma unroll
758 + for (int j = JCHUNK; j < 2 * JCHUNK; j++) {
759 + acc += to_float<scalar_t>(tmps[j]) * shared_exp_sums[j];
760 + }
761 + if (num_partitions > 2 * JCHUNK) {
762 + #pragma unroll
763 + for (int j = 2 * JCHUNK; j < MAX_NPAR; j++) {
764 + acc += to_float<scalar_t>(tmps[j]) * shared_exp_sums[j];
765 + }
766 + }
767 + }
768 +
769 + if (num_partitions > MAX_NPAR) {
770 + idx = 0;
771 + #pragma unroll
772 + for (int j = MAX_NPAR * HEAD_SIZE; j < 2 * MAX_NPAR * HEAD_SIZE;
773 + j += HEAD_SIZE) {
774 + // lastj is last valid partition
775 + const int lastj_offset =
776 + (j < num_partition_offset) ? j : last_partition_offset;
777 + tmps[idx] = tmp_out_ptr[lastj_offset];
778 + idx++;
779 + }
780 +
781 + #pragma unroll
782 + for (int j = 0; j < MAX_NPAR; j++) {
783 + acc += to_float<scalar_t>(tmps[j]) * shared_exp_sums[j + MAX_NPAR];
784 + }
785 + }
786 +
787 + const float inv_global_exp_sum =
788 + __fdividef(1.0f, shared_global_exp_sum + 1e-6f);
789 + acc *= inv_global_exp_sum;
790 + scalar_t* out_ptr =
791 + out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
792 + out_ptr[threadIdx.x] = from_float<scalar_t>(acc);
793 +}
794 +
795 +#else // !defined(__HIP__MI300_MI250__) TODO: Add NAVI support
796 +
797 +template <typename scalar_t, int BLOCK_SIZE, int HEAD_SIZE, int NUM_THREADS,
798 + int GQA_RATIO>
799 +__global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
800 + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
801 + const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
802 + // head_size/x, block_size, x]
803 + const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
804 + // head_size, block_size]
805 + const int num_kv_heads, const float scale,
806 + const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
807 + const int* __restrict__ context_lens, // [num_seqs]
808 + const int max_num_blocks_per_seq,
809 + const float* __restrict__ alibi_slopes, // [num_heads]
810 + const int q_stride, const int kv_block_stride, const int kv_head_stride,
811 + float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
812 + float* __restrict__ max_logits, // [num_seqs, num_heads,
813 + // max_num_partitions]
814 + scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions,
815 + // head_size]
816 + scalar_t* __restrict__ final_out, // [num_seqs, num_heads, head_size]
817 + #if 0
818 + scalar_t* __restrict__ qk_out, // [num_heads, num_seqs, max_ctx_blocks,block_size]
819 + #endif
820 + int max_ctx_blocks) {
821 + UNREACHABLE_CODE
822 +}
823 +
824 +// Grid: (num_heads, num_seqs).
825 +template <typename scalar_t, int HEAD_SIZE, int NUM_THREADS,
826 + int PARTITION_SIZE>
827 +__global__
828 +__launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
829 + scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
830 + const float* __restrict__ exp_sums, // [num_seqs, num_heads,
831 + // max_num_partitions]
832 + const float* __restrict__ max_logits, // [num_seqs, num_heads,
833 + // max_num_partitions]
834 + const scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads,
835 + // max_num_partitions, head_size]
836 + const int* __restrict__ context_lens, // [num_seqs]
837 + const int max_num_partitions){UNREACHABLE_CODE}
838 +
839 +#endif // defined(__HIP__MI300_MI250__) TODO: Add NAVI support
840 +
841 +#define LAUNCH_CUSTOM_ATTENTION(GQA_RATIO) \
842 + paged_attention_ll4mi_QKV_kernel<T, BLOCK_SIZE, HEAD_SIZE, NTHR, GQA_RATIO> \
843 + <<<grid, block, 0, stream>>>( \
844 + query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, scale, \
845 + block_tables_ptr, context_lens_ptr, max_num_blocks_per_seq, \
846 + alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
847 + exp_sums_ptr, max_logits_ptr, tmp_out_ptr, out_ptr, max_ctx_blocks);
848 +
849 +template <typename T, int BLOCK_SIZE, int HEAD_SIZE, int PARTITION_SIZE = 256>
850 +void paged_attention_custom_launcher(
851 + torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
852 + torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
853 + torch::Tensor& value_cache, const int num_kv_heads, float scale,
854 + torch::Tensor& block_tables, torch::Tensor& context_lens,
855 + int max_context_len,
856 +#if 0
857 + torch::Tensor& qk_out,
858 + torch::Tensor& softmax_out,
859 +#endif
860 + const c10::optional<torch::Tensor>& alibi_slopes) {
861 +
862 + int num_seqs = query.size(0);
863 + int num_heads = query.size(1);
864 + int head_size = query.size(2);
865 + int max_num_blocks_per_seq = block_tables.size(1);
866 + int q_stride = query.stride(0);
867 + int kv_block_stride = key_cache.stride(0);
868 + int kv_head_stride = key_cache.stride(1);
869 +
870 + // NOTE: alibi_slopes is optional.
871 + const float* alibi_slopes_ptr =
872 + alibi_slopes
873 + ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
874 + : nullptr;
875 +
876 + T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
877 + float* exp_sums_ptr = reinterpret_cast<float*>(exp_sums.data_ptr());
878 + float* max_logits_ptr = reinterpret_cast<float*>(max_logits.data_ptr());
879 + T* tmp_out_ptr = reinterpret_cast<T*>(tmp_out.data_ptr());
880 + T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
881 + T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
882 + T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
883 + int* block_tables_ptr = block_tables.data_ptr<int>();
884 + int* context_lens_ptr = context_lens.data_ptr<int>();
885 +#if 0
886 + T* qk_out_ptr = reinterpret_cast<T*>(qk_out.data_ptr());
887 + T* softmax_out_ptr = reinterpret_cast<T*>(softmax_out.data_ptr());
888 +#endif
889 +
890 + const int max_ctx_blocks = DIVIDE_ROUND_UP(max_context_len, BLOCK_SIZE);
891 + const int max_num_partitions =
892 + DIVIDE_ROUND_UP(max_context_len, PARTITION_SIZE);
893 + const int gqa_ratio = num_heads / num_kv_heads;
894 + assert(num_heads % num_kv_heads == 0);
895 + assert(head_size == HEAD_SIZE);
896 + assert(max_num_partitions <= 128);
897 +
898 + constexpr int NTHR = PARTITION_SIZE;
899 + dim3 grid(num_seqs, max_num_partitions, num_kv_heads);
900 + dim3 block(NTHR);
901 + const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
902 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
903 + switch (gqa_ratio) {
904 + case 1:
905 + LAUNCH_CUSTOM_ATTENTION(1);
906 + break;
907 + case 2:
908 + LAUNCH_CUSTOM_ATTENTION(2);
909 + break;
910 + case 3:
911 + LAUNCH_CUSTOM_ATTENTION(3);
912 + break;
913 + case 4:
914 + LAUNCH_CUSTOM_ATTENTION(4);
915 + break;
916 + case 5:
917 + LAUNCH_CUSTOM_ATTENTION(5);
918 + break;
919 + case 6:
920 + LAUNCH_CUSTOM_ATTENTION(6);
921 + break;
922 + case 7:
923 + LAUNCH_CUSTOM_ATTENTION(7);
924 + break;
925 + case 8:
926 + LAUNCH_CUSTOM_ATTENTION(8);
927 + break;
928 + case 9:
929 + LAUNCH_CUSTOM_ATTENTION(9);
930 + break;
931 + case 10:
932 + LAUNCH_CUSTOM_ATTENTION(10);
933 + break;
934 + case 11:
935 + LAUNCH_CUSTOM_ATTENTION(11);
936 + break;
937 + case 12:
938 + LAUNCH_CUSTOM_ATTENTION(12);
939 + break;
940 + case 13:
941 + LAUNCH_CUSTOM_ATTENTION(13);
942 + break;
943 + case 14:
944 + LAUNCH_CUSTOM_ATTENTION(14);
945 + break;
946 + case 15:
947 + LAUNCH_CUSTOM_ATTENTION(15);
948 + break;
949 + case 16:
950 + LAUNCH_CUSTOM_ATTENTION(16);
951 + break;
952 + default:
953 + TORCH_CHECK(false, "Unsupported gqa ratio: ", gqa_ratio);
954 + break;
955 + }
956 + // dim3 grid2(num_heads,num_seqs,head_size/HEAD_ELEMS_PER_WG);
957 + // dim3 block2(1024);
958 + // LAUNCH_CUSTOM_ATTENTION2;
959 +
960 + // reduction kernel is only required if max_context_len > partition size,
961 + // otherwise main kernel writes directly to final output
962 + // note there are cases with graphing where max_context_len is the max
963 + // supported by graphing, not the actual max among all the sequences: in that
964 + // case reduction kernel will still run but return immediately
965 + if (max_context_len > PARTITION_SIZE) {
966 + dim3 reduce_grid(num_heads, num_seqs);
967 + dim3 reduce_block(head_size);
968 + paged_attention_ll4mi_reduce_kernel<T, HEAD_SIZE, HEAD_SIZE, PARTITION_SIZE>
969 + <<<reduce_grid, reduce_block, 0, stream>>>(
970 + out_ptr, exp_sums_ptr, max_logits_ptr, tmp_out_ptr,
971 + context_lens_ptr, max_num_partitions);
972 + }
973 +}
974 +
975 +#define CALL_CUSTOM_LAUNCHER(T, BLK_SIZE, HEAD_SIZE) \
976 + paged_attention_custom_launcher<T, BLK_SIZE, HEAD_SIZE>( \
977 + out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
978 + num_kv_heads, scale, block_tables, context_lens, max_context_len, \
979 + alibi_slopes);
980 +
981 +#define CALL_CUSTOM_LAUNCHER_BLK(T, HEAD_SIZE) \
982 + switch (block_size) { \
983 + case 16: \
984 + CALL_CUSTOM_LAUNCHER(T, 16, HEAD_SIZE); \
985 + break; \
986 + case 32: \
987 + CALL_CUSTOM_LAUNCHER(T, 32, HEAD_SIZE); \
988 + break; \
989 + default: \
990 + TORCH_CHECK(false, "Unsupported block size: ", block_size); \
991 + break; \
992 + }
993 +
994 +#define CALL_CUSTOM_LAUNCHER_BLK_HEAD(T) \
995 + switch (head_size) { \
996 + case 64: \
997 + CALL_CUSTOM_LAUNCHER_BLK(T, 64); \
998 + break; \
999 + case 128: \
1000 + CALL_CUSTOM_LAUNCHER_BLK(T, 128); \
1001 + break; \
1002 + default: \
1003 + TORCH_CHECK(false, "Unsupported head size: ", head_size); \
1004 + break; \
1005 + }
1006 +
1007 +void paged_attention(
1008 + torch::Tensor& out, // [num_seqs, num_heads, head_size]
1009 + torch::Tensor& exp_sums, // [num_seqs, num_heads, max_num_partitions]
1010 + torch::Tensor& max_logits, // [num_seqs, num_heads, max_num_partitions]
1011 + torch::Tensor&
1012 + tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
1013 + torch::Tensor& query, // [num_seqs, num_heads, head_size]
1014 + torch::Tensor&
1015 + key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
1016 + torch::Tensor&
1017 + value_cache, // [num_blocks, num_heads, head_size, block_size]
1018 + int64_t num_kv_heads, double scale,
1019 + torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
1020 + torch::Tensor& context_lens, // [num_seqs]
1021 + int64_t block_size, int64_t max_context_len,
1022 + const c10::optional<torch::Tensor>& alibi_slopes,
1023 + const std::string& kv_cache_dtype) {
1024 + assert(kv_cache_dtype == "auto");
1025 + const int head_size = query.size(2);
1026 + if (query.dtype() == at::ScalarType::Half) {
1027 + CALL_CUSTOM_LAUNCHER_BLK_HEAD(_Float16);
1028 + } else if (query.dtype() == at::ScalarType::BFloat16) {
1029 + CALL_CUSTOM_LAUNCHER_BLK_HEAD(__hip_bfloat16);
1030 + } else {
1031 + TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
1032 + }
1033 +}
1034 +
1035 +#undef WARP_SIZE
1036 +#undef MAX
1037 +#undef MIN
1038 +#undef DIVIDE_ROUND_UP
csrc/rocm/ops.h new
+13
@@ -0,0 +1,13 @@
1 +#pragma once
2 +
3 +#include <torch/all.h>
4 +
5 +void paged_attention(torch::Tensor& out, torch::Tensor& exp_sums,
6 + torch::Tensor& max_logits, torch::Tensor& tmp_out,
7 + torch::Tensor& query, torch::Tensor& key_cache,
8 + torch::Tensor& value_cache, int64_t num_kv_heads,
9 + double scale, torch::Tensor& block_tables,
10 + torch::Tensor& context_lens, int64_t block_size,
11 + int64_t max_context_len,
12 + const c10::optional<torch::Tensor>& alibi_slopes,
13 + const std::string& kv_cache_dtype);
csrc/rocm/torch_bindings.cpp new
+33
@@ -0,0 +1,33 @@
1 +#include "core/registration.h"
2 +#include "rocm/ops.h"
3 +
4 +// Note on op signatures:
5 +// The X_meta signatures are for the meta functions corresponding to op X.
6 +// They must be kept in sync with the signature for X. Generally, only
7 +// functions that return Tensors require a meta function.
8 +//
9 +// See the following links for detailed docs on op registration and function
10 +// schemas.
11 +// https://docs.google.com/document/d/1_W62p8WJOQQUzPsJYa7s701JXt0qf2OfLub2sbkHOaU/edit#heading=h.ptttacy8y1u9
12 +// https://github.com/pytorch/pytorch/blob/main/aten/src/ATen/native/README.md#annotations
13 +
14 +TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, rocm_ops) {
15 + // vLLM custom ops for rocm
16 +
17 + // Custom attention op
18 + // Compute the attention between an input query and the cached
19 + // keys/values using PagedAttention.
20 + rocm_ops.def(
21 + "paged_attention(Tensor! out, Tensor exp_sums,"
22 + " Tensor max_logits, Tensor tmp_out,"
23 + " Tensor query, Tensor key_cache,"
24 + " Tensor value_cache, int num_kv_heads,"
25 + " float scale, Tensor block_tables,"
26 + " Tensor context_lens, int block_size,"
27 + " int max_context_len,"
28 + " Tensor? alibi_slopes,"
29 + " str kv_cache_dtype) -> ()");
30 + rocm_ops.impl("paged_attention", torch::kCUDA, &paged_attention);
31 +}
32 +
33 +REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
setup.py
+3
@@ -462,6 +462,9 @@ if _build_core_ext():
462 if _is_cuda() or _is_hip():
463 ext_modules.append(CMakeExtension(name="vllm._moe_C"))
464
465 +if _is_hip():
466 + ext_modules.append(CMakeExtension(name="vllm._rocm_C"))
467 +
468 if _build_custom_ops():
469 ext_modules.append(CMakeExtension(name="vllm._C"))
470
tests/kernels/test_attention.py
+164 -2
@@ -3,8 +3,6 @@ from typing import List, Optional, Tuple
3
4 import pytest
5 import torch
6 -from xformers import ops as xops
7 -from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
6
7 from tests.kernels.utils import opcheck
8 from vllm import _custom_ops as ops
@@ -12,6 +10,10 @@ from vllm.utils import get_max_shared_memory_bytes, is_hip
10
11 from .allclose_default import get_default_atol, get_default_rtol
12
13 +if not is_hip():
14 + from xformers import ops as xops
15 + from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
16 +
17 FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
18 # This will change depending on the compute capability.
19 # - 512 as a buffer
@@ -328,6 +330,165 @@ def ref_multi_query_kv_attention(
330 return torch.cat(ref_outputs, dim=0)
331
332
333 +@pytest.mark.parametrize("version", ["rocm"])
334 +@pytest.mark.parametrize("num_seqs", NUM_GEN_SEQS)
335 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
336 +@pytest.mark.parametrize("head_size", [64, 128]) # only test 64 128
337 +@pytest.mark.parametrize("use_alibi", USE_ALIBI)
338 +@pytest.mark.parametrize("block_size", BLOCK_SIZES)
339 +@pytest.mark.parametrize("dtype", DTYPES)
340 +@pytest.mark.parametrize("kv_cache_dtype", ["auto"])
341 +@pytest.mark.parametrize("seed", SEEDS)
342 +@pytest.mark.parametrize("device", CUDA_DEVICES)
343 +@pytest.mark.skipif(not is_hip(), reason="only for rocm")
344 +def test_paged_attention_rocm(
345 + kv_cache_factory,
346 + version: str,
347 + num_seqs: int,
348 + num_heads: Tuple[int, int],
349 + head_size: int,
350 + use_alibi: bool,
351 + block_size: int,
352 + dtype: torch.dtype,
353 + kv_cache_dtype: str,
354 + seed: int,
355 + device: str,
356 +) -> None:
357 + random.seed(seed)
358 + torch.random.manual_seed(seed)
359 + if torch.cuda.is_available():
360 + torch.cuda.manual_seed(seed)
361 + torch.set_default_device(device)
362 + scale = float(1.0 / (head_size**0.5))
363 + num_query_heads, num_kv_heads = num_heads
364 + query = torch.empty(num_seqs, num_query_heads, head_size, dtype=dtype)
365 + query.uniform_(-scale, scale)
366 +
367 + assert num_query_heads % num_kv_heads == 0
368 + num_queries_per_kv = num_query_heads // num_kv_heads
369 + alibi_slopes = None
370 + if use_alibi:
371 + alibi_slopes = torch.randn(num_query_heads, dtype=torch.float)
372 +
373 + context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
374 + context_lens[-1] = MAX_SEQ_LEN
375 + #context_lens = [8192 for _ in range(num_seqs)]
376 + max_context_len = max(context_lens)
377 + context_lens = torch.tensor(context_lens, dtype=torch.int)
378 + #print('>>> ctx lens', context_lens)
379 +
380 + # Create the block tables.
381 + max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
382 + block_tables = []
383 + for _ in range(num_seqs):
384 + block_table = [
385 + random.randint(0, NUM_BLOCKS - 1)
386 + for _ in range(max_num_blocks_per_seq)
387 + ]
388 + block_tables.append(block_table)
389 + block_tables = torch.tensor(block_tables, dtype=torch.int)
390 +
391 + # Create the KV caches.
392 + key_caches, value_caches = kv_cache_factory(NUM_BLOCKS, block_size, 1,
393 + num_kv_heads, head_size,
394 + kv_cache_dtype, dtype, seed,
395 + device)
396 + key_cache, value_cache = key_caches[0], value_caches[0]
397 +
398 + # TODO(charlifu) enable fp8 kv cache
399 + # Using default kv_scale
400 + # kv_scale = 1.0
401 +
402 + # Call the paged attention kernel.
403 + output = torch.empty_like(query)
404 + PARTITION_SIZE_ROCM = 256
405 + num_partitions = ((max_context_len + PARTITION_SIZE_ROCM - 1) //
406 + PARTITION_SIZE_ROCM)
407 + assert PARTITION_SIZE_ROCM % block_size == 0
408 + num_seqs, num_heads, head_size = output.shape
409 + tmp_output = torch.empty(
410 + size=(num_seqs, num_heads, num_partitions, head_size),
411 + dtype=output.dtype,
412 + )
413 + exp_sums = torch.empty(
414 + size=(num_seqs, num_heads, num_partitions),
415 + dtype=torch.float32,
416 + )
417 + max_logits = torch.empty_like(exp_sums)
418 + if version == "rocm":
419 + ops.paged_attention_rocm(
420 + output,
421 + exp_sums,
422 + max_logits,
423 + tmp_output,
424 + query,
425 + key_cache,
426 + value_cache,
427 + num_kv_heads,
428 + scale,
429 + block_tables,
430 + context_lens,
431 + block_size,
432 + max_context_len,
433 + alibi_slopes,
434 + kv_cache_dtype,
435 + )
436 + else:
437 + raise AssertionError(f"Unknown version: {version}")
438 +
439 + # Run the reference implementation.
440 + if kv_cache_dtype == "fp8":
441 + # Convert cache data back to dtype.
442 + x = 16 // torch.tensor([], dtype=dtype).element_size()
443 + key_cache_shape = (NUM_BLOCKS, num_kv_heads, head_size // x,
444 + block_size, x)
445 + dequantized_key_cache = torch.empty(size=key_cache_shape,
446 + dtype=dtype,
447 + device=device)
448 + ops.convert_fp8(key_cache, dequantized_key_cache)
449 + key_cache = dequantized_key_cache
450 +
451 + value_cache_shape = value_cache.shape
452 + dequantized_value_cache = torch.empty(size=value_cache_shape,
453 + dtype=dtype,
454 + device=device)
455 + ops.convert_fp8(value_cache, dequantized_value_cache)
456 + value_cache = dequantized_value_cache
457 +
458 + ref_output = torch.empty_like(query)
459 + ref_single_query_cached_kv_attention(
460 + ref_output,
461 + query,
462 + num_queries_per_kv,
463 + key_cache,
464 + value_cache,
465 + block_tables,
466 + context_lens,
467 + scale,
468 + alibi_slopes,
469 + )
470 +
471 + # NOTE(woosuk): Due to the kernel-level differences in the two
472 + # implementations, there is a small numerical difference in the two
473 + # outputs. Thus, we use a relaxed tolerance for the test.
474 + atol = get_default_atol(output) if is_hip() else 1e-3
475 + rtol = get_default_rtol(output) if is_hip() else 1e-5
476 +
477 + # NOTE(zhaoyang): FP8 KV Cache will introduce quantization error,
478 + # so we use a relaxed tolerance for the test.
479 + atol, rtol = 1e-4, 1e-5
480 + if dtype == torch.bfloat16:
481 + atol, rtol = 2e-4, 1e-5
482 + if use_alibi:
483 + if dtype == torch.half:
484 + atol, rtol = 5e-4, 1e-5
485 + if dtype == torch.bfloat16:
486 + atol, rtol = 1e-3, 1e-5
487 + if kv_cache_dtype == "fp8":
488 + atol, rtol = 1e-2, 1e-5
489 + assert torch.allclose(output, ref_output, atol=atol, rtol=rtol)
490 +
491 +
492 # TODO(woosuk): Add tests for USE_ALIBI=True.
493 @pytest.mark.parametrize("num_seqs", NUM_PREFILL_SEQS)
494 @pytest.mark.parametrize("num_heads", NUM_HEADS)
@@ -335,6 +496,7 @@ def ref_multi_query_kv_attention(
496 @pytest.mark.parametrize("dtype", DTYPES)
497 @pytest.mark.parametrize("seed", SEEDS)
498 @pytest.mark.parametrize("device", CUDA_DEVICES)
499 +@pytest.mark.skipif(is_hip(), reason="skip for rocm")
500 @torch.inference_mode()
501 def test_multi_query_kv_attention(
502 num_seqs: int,
vllm/_custom_ops.py
+27
@@ -17,6 +17,9 @@ if not current_platform.is_tpu():
17 except ImportError as e:
18 logger.warning("Failed to import from vllm._C with %r", e)
19
20 +if current_platform.is_rocm():
21 + import vllm._rocm_C # noqa: F401
22 +
23 with contextlib.suppress(ImportError):
24 import vllm._moe_C # noqa: F401
25
@@ -127,6 +130,30 @@ def paged_attention_v2(
130 blocksparse_block_size, blocksparse_head_sliding_step)
131
132
133 +def paged_attention_rocm(
134 + out: torch.Tensor,
135 + exp_sum: torch.Tensor,
136 + max_logits: torch.Tensor,
137 + tmp_out: torch.Tensor,
138 + query: torch.Tensor,
139 + key_cache: torch.Tensor,
140 + value_cache: torch.Tensor,
141 + num_kv_heads: int,
142 + scale: float,
143 + block_tables: torch.Tensor,
144 + seq_lens: torch.Tensor,
145 + block_size: int,
146 + max_seq_len: int,
147 + alibi_slopes: Optional[torch.Tensor],
148 + kv_cache_dtype: str,
149 +) -> None:
150 + torch.ops._rocm_C.paged_attention(out, exp_sum, max_logits, tmp_out, query,
151 + key_cache, value_cache, num_kv_heads,
152 + scale, block_tables, seq_lens,
153 + block_size, max_seq_len, alibi_slopes,
154 + kv_cache_dtype)
155 +
156 +
157 # pos encoding ops
158 def rotary_embedding(
159 positions: torch.Tensor,
vllm/attention/backends/rocm_flash_attn.py
+70 -14
@@ -5,6 +5,7 @@ from typing import Any, Dict, List, Optional, Tuple, Type
5 import torch
6
7 import vllm.envs as envs
8 +from vllm import _custom_ops as ops
9 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 AttentionMetadata, AttentionType)
11 from vllm.attention.backends.utils import (CommonAttentionState,
@@ -15,6 +16,9 @@ from vllm.logger import init_logger
16
17 logger = init_logger(__name__)
18
19 +_PARTITION_SIZE = 256
20 +ON_NAVI = "gfx1" in torch.cuda.get_device_properties("cuda").gcnArchName
21 +
22
23 class ROCmFlashAttentionBackend(AttentionBackend):
24
@@ -480,20 +484,61 @@ class ROCmFlashAttentionImpl(AttentionImpl):
484
485 if decode_meta := attn_metadata.decode_metadata:
486 # Decoding run.
483 - output[num_prefill_tokens:] = PagedAttention.forward_decode(
484 - decode_query,
485 - key_cache,
486 - value_cache,
487 - decode_meta.block_tables,
488 - decode_meta.seq_lens_tensor,
489 - decode_meta.max_decode_seq_len,
490 - self.kv_cache_dtype,
491 - self.num_kv_heads,
492 - self.scale,
493 - self.alibi_slopes,
494 - k_scale,
495 - v_scale,
496 - )
487 + # Whether to use rocm custom paged attention or not
488 + num_seqs, num_heads, head_size = decode_query.shape
489 + block_size = value_cache.shape[3]
490 + gqa_ratio = num_heads // self.num_kv_heads
491 + use_custom = use_rocm_custom_paged_attention(
492 + decode_query.dtype, head_size, block_size, self.kv_cache_dtype,
493 + gqa_ratio, decode_meta.max_decode_seq_len)
494 + if use_custom:
495 + max_seq_len = decode_meta.max_decode_seq_len
496 + max_num_partitions = ((max_seq_len + _PARTITION_SIZE - 1) //
497 + _PARTITION_SIZE)
498 + assert _PARTITION_SIZE % block_size == 0
499 + tmp_output = torch.empty(
500 + size=(num_seqs, num_heads, max_num_partitions, head_size),
501 + dtype=output.dtype,
502 + device=output.device,
503 + )
504 + exp_sums = torch.empty(
505 + size=(num_seqs, num_heads, max_num_partitions),
506 + dtype=torch.float32,
507 + device=output.device,
508 + )
509 + max_logits = torch.empty_like(exp_sums)
510 + ops.paged_attention_rocm(
511 + output[num_prefill_tokens:],
512 + exp_sums,
513 + max_logits,
514 + tmp_output,
515 + decode_query,
516 + key_cache,
517 + value_cache,
518 + self.num_kv_heads,
519 + self.scale,
520 + decode_meta.block_tables,
521 + decode_meta.seq_lens_tensor,
522 + block_size,
523 + max_seq_len,
524 + self.alibi_slopes,
525 + self.kv_cache_dtype,
526 + )
527 + else:
528 + output[num_prefill_tokens:] = PagedAttention.forward_decode(
529 + decode_query,
530 + key_cache,
531 + value_cache,
532 + decode_meta.block_tables,
533 + decode_meta.seq_lens_tensor,
534 + decode_meta.max_decode_seq_len,
535 + self.kv_cache_dtype,
536 + self.num_kv_heads,
537 + self.scale,
538 + self.alibi_slopes,
539 + k_scale,
540 + v_scale,
541 + )
542
543 # Reshape the output tensor.
544 return output.view(num_tokens, hidden_size)
@@ -532,3 +577,14 @@ def _sdpa_attention(
577 start = end
578
579 return output
580 +
581 +
582 +def use_rocm_custom_paged_attention(qtype: torch.dtype, head_size: int,
583 + block_size: int, kv_cache_dtype: str,
584 + gqa_ratio: int, max_seq_len: int) -> bool:
585 + # rocm custom page attention not support on navi (gfx1*)
586 + return (not ON_NAVI and (qtype == torch.half or qtype == torch.bfloat16)
587 + and (head_size == 64 or head_size == 128)
588 + and (block_size == 16 or block_size == 32)
589 + and kv_cache_dtype == "auto"
590 + and (gqa_ratio >= 1 and gqa_ratio <= 16) and max_seq_len <= 32768)