A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Kernel][Hardware][Amd]Custom paged attention kernel for rocm (#8310)
Charlie Fu committed
Sep 13, 2024 at 19:01 UTC
1ef0d2efd07f93bc7b0cfb597d8947b49e2fdaac
8 files changed
+1371
-16
CMakeLists.txt
+23
@@ -324,6 +324,25 @@ define_gpu_extension_target(
324
WITH_SOABI)
325
326
327
+if(VLLM_GPU_LANG STREQUAL "HIP")
328
+ #
329
+ # _rocm_C extension
330
+ #
331
+ set(VLLM_ROCM_EXT_SRC
332
+ "csrc/rocm/torch_bindings.cpp"
333
+ "csrc/rocm/attention.cu")
334
+
335
+ define_gpu_extension_target(
336
+ _rocm_C
337
+ DESTINATION vllm
338
+ LANGUAGE ${VLLM_GPU_LANG}
339
+ SOURCES ${VLLM_ROCM_EXT_SRC}
340
+ COMPILE_FLAGS ${VLLM_GPU_FLAGS}
341
+ ARCHITECTURES ${VLLM_GPU_ARCHES}
342
+ USE_SABI 3
343
+ WITH_SOABI)
344
+endif()
345
+
346
347
if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
348
message(STATUS "Enabling C extension.")
@@ -331,5 +350,9 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
350
351
message(STATUS "Enabling moe extension.")
352
add_dependencies(default _moe_C)
353
+endif()
354
355
+if(VLLM_GPU_LANG STREQUAL "HIP")
356
+ message(STATUS "Enabling rocm extension.")
357
+ add_dependencies(default _rocm_C)
358
endif()
csrc/rocm/attention.cu
new
+1038
@@ -0,0 +1,1038 @@
1
+/*
2
+ * Copyright (c) 2024, The vLLM team.
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * http://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+
17
+#include <torch/all.h>
18
+#include <ATen/cuda/CUDAContext.h>
19
+#include <c10/cuda/CUDAGuard.h>
20
+#include <hip/hip_bf16.h>
21
+
22
+#include <algorithm>
23
+
24
+#if defined(__HIPCC__) && (defined(__gfx90a__) || defined(__gfx940__) || \
25
+ defined(__gfx941__) || defined(__gfx942__))
26
+ #define __HIP__MI300_MI250__
27
+#endif
28
+
29
+#if defined(NDEBUG)
30
+ #undef NDEBUG
31
+ #include <assert.h>
32
+ #define UNREACHABLE_CODE assert(false);
33
+ #define NDEBUG
34
+#else
35
+ #define UNREACHABLE_CODE assert(false);
36
+#endif
37
+
38
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
39
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
40
+#define DIVIDE_ROUND_UP(a, b) (((a) + (b) - 1) / (b))
41
+#define WARP_SIZE 64
42
+
43
+#if defined(__HIP__MI300_MI250__) // TODO: Add NAVI support
44
+
45
+ #define GCN_MFMA_INSTR1 __builtin_amdgcn_mfma_f32_16x16x4f32
46
+ #define GCN_MFMA_INSTR __builtin_amdgcn_mfma_f32_4x4x4f16
47
+
48
+using floatx4 = __attribute__((__vector_size__(4 * sizeof(float)))) float;
49
+using float16x4 =
50
+ __attribute__((__vector_size__(4 * sizeof(_Float16)))) _Float16;
51
+typedef float16x4 _Half4;
52
+typedef struct _Half8 {
53
+ _Half4 xy[2];
54
+} _Half8;
55
+
56
+using bit16_t = uint16_t;
57
+using bit16x4 = __attribute__((__vector_size__(4 * sizeof(uint16_t)))) uint16_t;
58
+typedef bit16x4 _B16x4;
59
+typedef struct _B16x8 {
60
+ _B16x4 xy[2];
61
+} _B16x8;
62
+
63
+////// Non temporal load stores ///////
64
+
65
+template <typename T>
66
+__device__ __forceinline__ T load(T* addr) {
67
+ return addr[0];
68
+}
69
+
70
+template <typename T>
71
+__device__ __forceinline__ void store(T value, T* addr) {
72
+ addr[0] = value;
73
+}
74
+
75
+template <typename T, int absz, int cbid, int blgp>
76
+__device__ __forceinline__ floatx4 gcn_mfma_instr(const _B16x4& inpA,
77
+ const _B16x4& inpB,
78
+ const floatx4& inpC) {
79
+ if constexpr (std::is_same<T, _Float16>::value) {
80
+ return __builtin_amdgcn_mfma_f32_4x4x4f16(inpA, inpB, inpC, absz, cbid,
81
+ blgp);
82
+ } else if constexpr (std::is_same<T, __hip_bfloat16>::value) {
83
+ return __builtin_amdgcn_mfma_f32_4x4x4bf16_1k(inpA, inpB, inpC, absz, cbid,
84
+ blgp);
85
+ } else {
86
+ static_assert(false, "unsupported 16b dtype");
87
+ }
88
+}
89
+
90
+template <typename T>
91
+__device__ __forceinline__ float to_float(const T& inp) {
92
+ if constexpr (std::is_same<T, _Float16>::value) {
93
+ return (float)inp;
94
+ } else if constexpr (std::is_same<T, __hip_bfloat16>::value) {
95
+ return __bfloat162float(inp);
96
+ } else {
97
+ static_assert(false, "unsupported 16b dtype");
98
+ }
99
+}
100
+
101
+template <typename T>
102
+__device__ __forceinline__ T from_float(const float& inp) {
103
+ if constexpr (std::is_same<T, _Float16>::value) {
104
+ return (_Float16)inp;
105
+ } else if constexpr (std::is_same<T, __hip_bfloat16>::value) {
106
+ return __float2bfloat16(inp);
107
+ } else {
108
+ static_assert(false, "unsupported 16b dtype");
109
+ }
110
+}
111
+
112
+template <typename T>
113
+__device__ __forceinline__ _B16x4 from_floatx4(const floatx4& inp) {
114
+ union tmpcvt {
115
+ uint16_t u;
116
+ _Float16 f;
117
+ __hip_bfloat16 b;
118
+ } t16;
119
+ _B16x4 ret;
120
+ if constexpr (std::is_same<T, _Float16>::value) {
121
+ #pragma unroll
122
+ for (int i = 0; i < 4; i++) {
123
+ t16.f = (_Float16)inp[i];
124
+ ret[i] = t16.u;
125
+ }
126
+ return ret;
127
+ } else if constexpr (std::is_same<T, __hip_bfloat16>::value) {
128
+ #pragma unroll
129
+ for (int i = 0; i < 4; i++) {
130
+ t16.b = __float2bfloat16(inp[i]);
131
+ ret[i] = t16.u;
132
+ }
133
+ return ret;
134
+ } else {
135
+ static_assert(false, "unsupported 16b dtype");
136
+ }
137
+}
138
+
139
+template <typename T>
140
+__device__ __forceinline__ _B16x4 addx4(const _B16x4& inp1,
141
+ const _B16x4& inp2) {
142
+ union tmpcvt {
143
+ uint16_t u;
144
+ _Float16 f;
145
+ __hip_bfloat16 b;
146
+ } t1, t2, res;
147
+ _B16x4 ret;
148
+ if constexpr (std::is_same<T, _Float16>::value) {
149
+ #pragma unroll
150
+ for (int i = 0; i < 4; i++) {
151
+ t1.u = inp1[i];
152
+ t2.u = inp2[i];
153
+ res.f = t1.f + t2.f;
154
+ ret[i] = res.u;
155
+ }
156
+ return ret;
157
+ } else if constexpr (std::is_same<T, __hip_bfloat16>::value) {
158
+ #pragma unroll
159
+ for (int i = 0; i < 4; i++) {
160
+ t1.u = inp1[i];
161
+ t2.u = inp2[i];
162
+ res.b = t1.b + t2.b;
163
+ ret[i] = res.u;
164
+ }
165
+ return ret;
166
+ } else {
167
+ static_assert(false, "unsupported 16b dtype");
168
+ }
169
+}
170
+
171
+///////////////////////////////////////
172
+
173
+// grid (num_seqs, num_partitions,num_heads/gqa_ratio)
174
+// block (partition size)
175
+template <typename scalar_t, int BLOCK_SIZE, int HEAD_SIZE, int NUM_THREADS,
176
+ int GQA_RATIO>
177
+__global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
178
+ const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
179
+ const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
180
+ // head_size/x, block_size, x]
181
+ const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
182
+ // head_size, block_size]
183
+ const int num_kv_heads, const float scale,
184
+ const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
185
+ const int* __restrict__ context_lens, // [num_seqs]
186
+ const int max_num_blocks_per_seq,
187
+ const float* __restrict__ alibi_slopes, // [num_heads]
188
+ const int q_stride, const int kv_block_stride, const int kv_head_stride,
189
+ float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
190
+ float* __restrict__ max_logits, // [num_seqs, num_heads,
191
+ // max_num_partitions]
192
+ scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions,
193
+ // head_size]
194
+ scalar_t* __restrict__ final_out, // [num_seqs, num_heads, head_size]
195
+ #if 0
196
+ scalar_t* __restrict__ qk_out, // [num_heads, num_seqs, max_ctx_blocks,block_size]
197
+ #endif
198
+ int max_ctx_blocks) {
199
+ constexpr int NWARPS = NUM_THREADS / WARP_SIZE;
200
+ const int warpid = threadIdx.x / WARP_SIZE;
201
+ const int laneid = threadIdx.x % WARP_SIZE;
202
+ const int lane4id = laneid % 4;
203
+
204
+ const int seq_idx = blockIdx.x;
205
+ const int partition_idx = blockIdx.y;
206
+ const int partition_size = blockDim.x;
207
+ const int max_num_partitions = gridDim.y;
208
+
209
+ const int context_len = context_lens[seq_idx];
210
+ const int partition_start_token_idx = partition_idx * partition_size;
211
+ // exit if partition is out of context for seq
212
+ if (partition_start_token_idx >= context_len) {
213
+ return;
214
+ }
215
+ constexpr int QHLOOP =
216
+ DIVIDE_ROUND_UP(GQA_RATIO, 4); // each 4 lanes fetch 4 different qheads,
217
+ // total qheads =8, so qhloop is 2
218
+ constexpr int GQA_RATIO4 = 4 * QHLOOP;
219
+ __shared__ float shared_qk_max[NWARPS][GQA_RATIO4 + 1];
220
+ __shared__ float shared_exp_sum[NWARPS][GQA_RATIO4 + 1];
221
+ _B16x8 Qlocal[QHLOOP];
222
+ constexpr int x = 16 / sizeof(scalar_t);
223
+ constexpr int KHELOOP = HEAD_SIZE / x;
224
+ _B16x8 Klocal[KHELOOP];
225
+ constexpr int VHELOOP =
226
+ HEAD_SIZE /
227
+ WARP_SIZE; // v head_size dimension is distributed across lanes
228
+ constexpr int VTLOOP = 8; // 16 separate 4xtokens across warp -> 16/2
229
+ // 8xtokens
230
+ _B16x8 Vlocal[VHELOOP][VTLOOP];
231
+ floatx4 dout[QHLOOP];
232
+ float qk_max[QHLOOP];
233
+ #pragma unroll
234
+ for (int h = 0; h < QHLOOP; h++) {
235
+ dout[h] = {0};
236
+ qk_max[h] = -FLT_MAX;
237
+ }
238
+
239
+ const int wg_start_head_idx = blockIdx.z * GQA_RATIO;
240
+ const int wg_start_kv_head_idx = blockIdx.z;
241
+
242
+ const int warp_start_token_idx =
243
+ partition_start_token_idx + warpid * WARP_SIZE;
244
+
245
+ if (warp_start_token_idx >= context_len) { // warp out of context
246
+ #pragma unroll
247
+ for (int h = 0; h < GQA_RATIO4; h++) {
248
+ shared_qk_max[warpid][h] = -FLT_MAX;
249
+ shared_exp_sum[warpid][h] = 0.0f;
250
+ }
251
+ } else { // warp within context
252
+
253
+ const int num_context_blocks = DIVIDE_ROUND_UP(context_len, BLOCK_SIZE);
254
+ const int last_ctx_block = num_context_blocks - 1;
255
+
256
+ const int* block_table = block_tables + seq_idx * max_num_blocks_per_seq;
257
+
258
+ const int local_token_idx = threadIdx.x;
259
+ const int global_token_idx = partition_start_token_idx + local_token_idx;
260
+
261
+ const int block_idx = (global_token_idx < context_len)
262
+ ? global_token_idx / BLOCK_SIZE
263
+ : last_ctx_block;
264
+ // fetch block number for q and k
265
+ // int32 physical_block_number leads to overflow when multiplied with
266
+ // kv_block_stride
267
+ const int64_t physical_block_number =
268
+ static_cast<int64_t>(block_table[block_idx]);
269
+
270
+ // fetch vphysical block numbers up front
271
+ constexpr int VBLOCKS = 8 * VTLOOP / BLOCK_SIZE;
272
+ int vphysical_blocks[VBLOCKS];
273
+
274
+ const int warp_start_block_idx = warp_start_token_idx / BLOCK_SIZE;
275
+ #pragma unroll
276
+ for (int b = 0; b < VBLOCKS; b++) {
277
+ const int vblock_idx = warp_start_block_idx + b;
278
+ const int vblock_idx_ctx =
279
+ (vblock_idx <= last_ctx_block) ? vblock_idx : last_ctx_block;
280
+ vphysical_blocks[b] = block_table[vblock_idx_ctx];
281
+ }
282
+ // each 4 lanes fetch 8 helems, so warp fetches 8*16 = 128 helems
283
+ const scalar_t* q_ptr =
284
+ q + seq_idx * q_stride + wg_start_head_idx * HEAD_SIZE;
285
+ const _B16x8* q_ptrh8 = reinterpret_cast<const _B16x8*>(q_ptr);
286
+ const int qhead_elemh8 = laneid / 4;
287
+ #pragma unroll
288
+ for (int h = 0; h < QHLOOP - 1; h++) {
289
+ const int qhead_idx = h * 4 + lane4id;
290
+ Qlocal[h] = q_ptrh8[qhead_idx * HEAD_SIZE / 8 + qhead_elemh8];
291
+ }
292
+ const int final_qhead_idx = 4 * (QHLOOP - 1) + lane4id;
293
+ if (final_qhead_idx < GQA_RATIO) {
294
+ Qlocal[QHLOOP - 1] =
295
+ q_ptrh8[final_qhead_idx * HEAD_SIZE / 8 + qhead_elemh8];
296
+ } else {
297
+ Qlocal[QHLOOP - 1].xy[0] = {0};
298
+ Qlocal[QHLOOP - 1].xy[1] = {0};
299
+ }
300
+
301
+ const scalar_t* k_ptr = k_cache + physical_block_number * kv_block_stride +
302
+ wg_start_kv_head_idx * kv_head_stride;
303
+
304
+ const int physical_block_offset =
305
+ local_token_idx % BLOCK_SIZE; // since x=half8, physical_block_offset
306
+ // is already cast as _H8
307
+
308
+ const _B16x8* k_ptrh8 = reinterpret_cast<const _B16x8*>(k_ptr);
309
+ #pragma unroll
310
+ for (int d = 0; d < KHELOOP; d++) {
311
+ Klocal[d] = k_ptrh8[d * BLOCK_SIZE + physical_block_offset];
312
+ }
313
+
314
+ float alibi_slope[QHLOOP];
315
+ if (alibi_slopes != nullptr) {
316
+ #pragma unroll
317
+ for (int h = 0; h < QHLOOP; h++) {
318
+ const int qhead_idx = h * 4 + lane4id;
319
+ alibi_slope[h] = (qhead_idx < GQA_RATIO)
320
+ ? alibi_slopes[wg_start_head_idx + qhead_idx]
321
+ : 0.f;
322
+ }
323
+ }
324
+
325
+ const scalar_t* v_ptr = v_cache + wg_start_kv_head_idx * kv_head_stride;
326
+ const _B16x8* v_ptrh8 = reinterpret_cast<const _B16x8*>(v_ptr);
327
+ // iterate over each v block
328
+ #pragma unroll
329
+ for (int b = 0; b < VBLOCKS; b++) {
330
+ // int32 physical_block_number leads to overflow when multiplied with
331
+ // kv_block_stride
332
+ const int64_t vphysical_block_number =
333
+ static_cast<int64_t>(vphysical_blocks[b]);
334
+ const _B16x8* v_ptrh8b =
335
+ v_ptrh8 + (vphysical_block_number * kv_block_stride) / 8;
336
+ // iterate over each head elem (within head_size)
337
+ #pragma unroll
338
+ for (int h = 0; h < VHELOOP; h++) {
339
+ const int head_size_elem = h * WARP_SIZE + laneid;
340
+ const _B16x8* v_ptrh8be = v_ptrh8b + head_size_elem * BLOCK_SIZE / 8;
341
+ // iterate over all velems within block
342
+ #pragma unroll
343
+ for (int d = 0; d < BLOCK_SIZE / 8; d++) {
344
+ Vlocal[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
345
+ }
346
+ }
347
+ }
348
+
349
+ #pragma unroll
350
+ for (int h = 0; h < QHLOOP; h++) {
351
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 0, 0>(Qlocal[h].xy[0],
352
+ Klocal[0].xy[0], dout[h]);
353
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 0, 0>(Qlocal[h].xy[1],
354
+ Klocal[0].xy[1], dout[h]);
355
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 1, 0>(Qlocal[h].xy[0],
356
+ Klocal[1].xy[0], dout[h]);
357
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 1, 0>(Qlocal[h].xy[1],
358
+ Klocal[1].xy[1], dout[h]);
359
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 2, 0>(Qlocal[h].xy[0],
360
+ Klocal[2].xy[0], dout[h]);
361
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 2, 0>(Qlocal[h].xy[1],
362
+ Klocal[2].xy[1], dout[h]);
363
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 3, 0>(Qlocal[h].xy[0],
364
+ Klocal[3].xy[0], dout[h]);
365
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 3, 0>(Qlocal[h].xy[1],
366
+ Klocal[3].xy[1], dout[h]);
367
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 4, 0>(Qlocal[h].xy[0],
368
+ Klocal[4].xy[0], dout[h]);
369
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 4, 0>(Qlocal[h].xy[1],
370
+ Klocal[4].xy[1], dout[h]);
371
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 5, 0>(Qlocal[h].xy[0],
372
+ Klocal[5].xy[0], dout[h]);
373
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 5, 0>(Qlocal[h].xy[1],
374
+ Klocal[5].xy[1], dout[h]);
375
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 6, 0>(Qlocal[h].xy[0],
376
+ Klocal[6].xy[0], dout[h]);
377
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 6, 0>(Qlocal[h].xy[1],
378
+ Klocal[6].xy[1], dout[h]);
379
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 7, 0>(Qlocal[h].xy[0],
380
+ Klocal[7].xy[0], dout[h]);
381
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 7, 0>(Qlocal[h].xy[1],
382
+ Klocal[7].xy[1], dout[h]);
383
+ if constexpr (KHELOOP > 8) {
384
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 8, 0>(Qlocal[h].xy[0],
385
+ Klocal[8].xy[0], dout[h]);
386
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 8, 0>(Qlocal[h].xy[1],
387
+ Klocal[8].xy[1], dout[h]);
388
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 9, 0>(Qlocal[h].xy[0],
389
+ Klocal[9].xy[0], dout[h]);
390
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 9, 0>(Qlocal[h].xy[1],
391
+ Klocal[9].xy[1], dout[h]);
392
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 10, 0>(Qlocal[h].xy[0],
393
+ Klocal[10].xy[0], dout[h]);
394
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 10, 0>(Qlocal[h].xy[1],
395
+ Klocal[10].xy[1], dout[h]);
396
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 11, 0>(Qlocal[h].xy[0],
397
+ Klocal[11].xy[0], dout[h]);
398
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 11, 0>(Qlocal[h].xy[1],
399
+ Klocal[11].xy[1], dout[h]);
400
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 12, 0>(Qlocal[h].xy[0],
401
+ Klocal[12].xy[0], dout[h]);
402
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 12, 0>(Qlocal[h].xy[1],
403
+ Klocal[12].xy[1], dout[h]);
404
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 13, 0>(Qlocal[h].xy[0],
405
+ Klocal[13].xy[0], dout[h]);
406
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 13, 0>(Qlocal[h].xy[1],
407
+ Klocal[13].xy[1], dout[h]);
408
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 14, 0>(Qlocal[h].xy[0],
409
+ Klocal[14].xy[0], dout[h]);
410
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 14, 0>(Qlocal[h].xy[1],
411
+ Klocal[14].xy[1], dout[h]);
412
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 15, 0>(Qlocal[h].xy[0],
413
+ Klocal[15].xy[0], dout[h]);
414
+ dout[h] = gcn_mfma_instr<scalar_t, 4, 15, 0>(Qlocal[h].xy[1],
415
+ Klocal[15].xy[1], dout[h]);
416
+ } // KHELOOP>8
417
+ dout[h] *= scale;
418
+ }
419
+ // transpose dout so that 4 token ids are in each lane, and 4 heads are across
420
+ // 4 lanes
421
+ #pragma unroll
422
+ for (int h = 0; h < QHLOOP; h++) {
423
+ floatx4 tmp = {0};
424
+ #pragma unroll
425
+ for (int i = 0; i < 4; i++) {
426
+ const float B = (lane4id == i) ? 1.0f : 0.0f;
427
+ // const float A = (global_token_idx < context_len) ? dout[h][i] : 0.0f;
428
+ tmp = __builtin_amdgcn_mfma_f32_4x4x1f32(dout[h][i], B, tmp, 0, 0, 0);
429
+ // tmp = __builtin_amdgcn_mfma_f32_4x4x1f32(A, B, tmp, 0, 0, 0);
430
+ }
431
+ dout[h] = tmp;
432
+ }
433
+
434
+ const int lane4_token_idx = 4 * (global_token_idx >> 2);
435
+ const int alibi_offset = lane4_token_idx - context_len + 1;
436
+ if (alibi_slopes != nullptr) {
437
+ #pragma unroll
438
+ for (int h = 0; h < QHLOOP; h++) {
439
+ #pragma unroll
440
+ for (int i = 0; i < 4; i++) {
441
+ dout[h][i] += alibi_slope[h] * (alibi_offset + i);
442
+ }
443
+ }
444
+ }
445
+
446
+ #pragma unroll
447
+ for (int h = 0; h < QHLOOP; h++) {
448
+ qk_max[h] = -FLT_MAX;
449
+ #pragma unroll
450
+ for (int i = 0; i < 4; i++) {
451
+ qk_max[h] = (lane4_token_idx + i < context_len)
452
+ ? fmaxf(qk_max[h], dout[h][i])
453
+ : qk_max[h];
454
+ }
455
+ #pragma unroll
456
+ for (int mask = WARP_SIZE / 2; mask >= 4; mask /= 2) {
457
+ qk_max[h] = fmaxf(qk_max[h], __shfl_xor(qk_max[h], mask));
458
+ }
459
+ }
460
+
461
+ float exp_sum[QHLOOP];
462
+ #pragma unroll
463
+ for (int h = 0; h < QHLOOP; h++) {
464
+ exp_sum[h] = 0.0f;
465
+ #pragma unroll
466
+ for (int i = 0; i < 4; i++) {
467
+ dout[h][i] = (lane4_token_idx + i < context_len)
468
+ ? __expf(dout[h][i] - qk_max[h])
469
+ : 0.0f;
470
+ exp_sum[h] += dout[h][i];
471
+ }
472
+ #pragma unroll
473
+ for (int mask = WARP_SIZE / 2; mask >= 4; mask /= 2) {
474
+ exp_sum[h] += __shfl_xor(exp_sum[h], mask);
475
+ }
476
+ }
477
+
478
+ #pragma unroll
479
+ for (int h = 0; h < QHLOOP; h++) {
480
+ const int head_idx = 4 * h + lane4id;
481
+ shared_qk_max[warpid][head_idx] = qk_max[h];
482
+ shared_exp_sum[warpid][head_idx] = exp_sum[h];
483
+ }
484
+ } // warp within context
485
+
486
+ __syncthreads();
487
+
488
+ const int num_heads = gridDim.z * GQA_RATIO;
489
+ float* max_logits_ptr =
490
+ max_logits + seq_idx * num_heads * max_num_partitions + partition_idx;
491
+ float* exp_sums_ptr =
492
+ exp_sums + seq_idx * num_heads * max_num_partitions + partition_idx;
493
+ #pragma unroll
494
+ for (int h = 0; h < QHLOOP; h++) {
495
+ float global_qk_max = -FLT_MAX;
496
+ float warp_qk_max[NWARPS];
497
+ const int head_idx = 4 * h + lane4id;
498
+ #pragma unroll
499
+ for (int w = 0; w < NWARPS; w++) {
500
+ warp_qk_max[w] = shared_qk_max[w][head_idx];
501
+ global_qk_max = fmaxf(global_qk_max, warp_qk_max[w]);
502
+ }
503
+ float global_exp_sum = 0.0f;
504
+ #pragma unroll
505
+ for (int w = 0; w < NWARPS; w++) {
506
+ global_exp_sum +=
507
+ shared_exp_sum[w][head_idx] * __expf(warp_qk_max[w] - global_qk_max);
508
+ }
509
+ if (head_idx < GQA_RATIO) {
510
+ max_logits_ptr[(wg_start_head_idx + head_idx) * max_num_partitions] =
511
+ global_qk_max;
512
+ exp_sums_ptr[(wg_start_head_idx + head_idx) * max_num_partitions] =
513
+ global_exp_sum;
514
+ }
515
+ const float global_inv_sum_scale = __fdividef(1.f, global_exp_sum + 1e-6f) *
516
+ __expf(qk_max[h] - global_qk_max);
517
+ dout[h] *= global_inv_sum_scale;
518
+ }
519
+ // logits[h] -> every 4 lanes hold 4 heads, each lane holds 4 tokens, there
520
+ // are 4x16 tokens across warp
521
+ _B16x4 logits[QHLOOP];
522
+ #pragma unroll
523
+ for (int h = 0; h < QHLOOP; h++) {
524
+ logits[h] = from_floatx4<scalar_t>(dout[h]);
525
+ }
526
+
527
+ __shared__ _B16x4 vout_shared[QHLOOP][VHELOOP][WARP_SIZE][NWARPS + 1];
528
+
529
+ if (warp_start_token_idx >= context_len) { // warp out of context
530
+ #pragma unroll
531
+ for (int qh = 0; qh < QHLOOP; qh++) {
532
+ #pragma unroll
533
+ for (int vh = 0; vh < VHELOOP; vh++) {
534
+ vout_shared[qh][vh][laneid][warpid] = {0};
535
+ }
536
+ }
537
+ } else { // warp in context
538
+ // iterate across heads
539
+ #pragma unroll
540
+ for (int qh = 0; qh < QHLOOP; qh++) {
541
+ // iterate over each v head elem (within head_size)
542
+ #pragma unroll
543
+ for (int vh = 0; vh < VHELOOP; vh++) {
544
+ floatx4 acc = {0};
545
+ // iterate over tokens
546
+ acc = gcn_mfma_instr<scalar_t, 4, 0, 0>(logits[qh], Vlocal[vh][0].xy[0],
547
+ acc);
548
+ acc = gcn_mfma_instr<scalar_t, 4, 1, 0>(logits[qh], Vlocal[vh][0].xy[1],
549
+ acc);
550
+ acc = gcn_mfma_instr<scalar_t, 4, 2, 0>(logits[qh], Vlocal[vh][1].xy[0],
551
+ acc);
552
+ acc = gcn_mfma_instr<scalar_t, 4, 3, 0>(logits[qh], Vlocal[vh][1].xy[1],
553
+ acc);
554
+ acc = gcn_mfma_instr<scalar_t, 4, 4, 0>(logits[qh], Vlocal[vh][2].xy[0],
555
+ acc);
556
+ acc = gcn_mfma_instr<scalar_t, 4, 5, 0>(logits[qh], Vlocal[vh][2].xy[1],
557
+ acc);
558
+ acc = gcn_mfma_instr<scalar_t, 4, 6, 0>(logits[qh], Vlocal[vh][3].xy[0],
559
+ acc);
560
+ acc = gcn_mfma_instr<scalar_t, 4, 7, 0>(logits[qh], Vlocal[vh][3].xy[1],
561
+ acc);
562
+ acc = gcn_mfma_instr<scalar_t, 4, 8, 0>(logits[qh], Vlocal[vh][4].xy[0],
563
+ acc);
564
+ acc = gcn_mfma_instr<scalar_t, 4, 9, 0>(logits[qh], Vlocal[vh][4].xy[1],
565
+ acc);
566
+ acc = gcn_mfma_instr<scalar_t, 4, 10, 0>(logits[qh],
567
+ Vlocal[vh][5].xy[0], acc);
568
+ acc = gcn_mfma_instr<scalar_t, 4, 11, 0>(logits[qh],
569
+ Vlocal[vh][5].xy[1], acc);
570
+ acc = gcn_mfma_instr<scalar_t, 4, 12, 0>(logits[qh],
571
+ Vlocal[vh][6].xy[0], acc);
572
+ acc = gcn_mfma_instr<scalar_t, 4, 13, 0>(logits[qh],
573
+ Vlocal[vh][6].xy[1], acc);
574
+ acc = gcn_mfma_instr<scalar_t, 4, 14, 0>(logits[qh],
575
+ Vlocal[vh][7].xy[0], acc);
576
+ acc = gcn_mfma_instr<scalar_t, 4, 15, 0>(logits[qh],
577
+ Vlocal[vh][7].xy[1], acc);
578
+ vout_shared[qh][vh][laneid][warpid] = from_floatx4<scalar_t>(acc);
579
+ }
580
+ }
581
+ } // warp in context
582
+
583
+ __syncthreads();
584
+
585
+ if (warpid == 0) {
586
+ _B16x4 vout[QHLOOP][VHELOOP];
587
+ // iterate across heads
588
+ scalar_t* out_ptr;
589
+ int out_num_partitions;
590
+ if (context_len > partition_size) {
591
+ out_num_partitions = max_num_partitions;
592
+ out_ptr = out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE +
593
+ partition_idx * HEAD_SIZE;
594
+ } else {
595
+ out_num_partitions = 1;
596
+ out_ptr = final_out + seq_idx * num_heads * HEAD_SIZE;
597
+ }
598
+ #pragma unroll
599
+ for (int qh = 0; qh < QHLOOP; qh++) {
600
+ // iterate over each v head elem (within head_size)
601
+ #pragma unroll
602
+ for (int vh = 0; vh < VHELOOP; vh++) {
603
+ vout[qh][vh] = {0};
604
+ #pragma unroll
605
+ for (int w = 0; w < NWARPS; w++) {
606
+ vout[qh][vh] =
607
+ addx4<scalar_t>(vout[qh][vh], vout_shared[qh][vh][laneid][w]);
608
+ }
609
+ const int head_size_elem = vh * WARP_SIZE + laneid;
610
+ bit16_t* out_ptr_b16 = reinterpret_cast<bit16_t*>(out_ptr);
611
+ #pragma unroll
612
+ for (int i = 0; i < 4; i++) {
613
+ const int head_idx = 4 * qh + i;
614
+ if (head_idx < GQA_RATIO) {
615
+ out_ptr_b16[(wg_start_head_idx + head_idx) * out_num_partitions *
616
+ HEAD_SIZE +
617
+ head_size_elem] = vout[qh][vh][i];
618
+ }
619
+ }
620
+ }
621
+ }
622
+ }
623
+}
624
+
625
+// Grid: (num_heads, num_seqs).
626
+template <typename scalar_t, int HEAD_SIZE, int NUM_THREADS,
627
+ int PARTITION_SIZE>
628
+__global__
629
+__launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
630
+ scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
631
+ const float* __restrict__ exp_sums, // [num_seqs, num_heads,
632
+ // max_num_partitions]
633
+ const float* __restrict__ max_logits, // [num_seqs, num_heads,
634
+ // max_num_partitions]
635
+ const scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads,
636
+ // max_num_partitions, head_size]
637
+ const int* __restrict__ context_lens, // [num_seqs]
638
+ const int max_num_partitions) {
639
+ const int num_heads = gridDim.x;
640
+ const int head_idx = blockIdx.x;
641
+ const int seq_idx = blockIdx.y;
642
+ const int context_len = context_lens[seq_idx];
643
+ const int num_partitions = DIVIDE_ROUND_UP(context_len, PARTITION_SIZE);
644
+ if (num_partitions == 1) {
645
+ // if num_partitions==1, main kernel will write to out directly, no work in
646
+ // reduction kernel
647
+ return;
648
+ }
649
+
650
+ constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
651
+ const int warpid = threadIdx.x / WARP_SIZE;
652
+ const int laneid = threadIdx.x % WARP_SIZE;
653
+
654
+ __shared__ float shared_global_exp_sum;
655
+ __shared__ float shared_exp_sums[2 * WARP_SIZE];
656
+
657
+ if (warpid == 0) {
658
+ const float* max_logits_ptr = max_logits +
659
+ seq_idx * num_heads * max_num_partitions +
660
+ head_idx * max_num_partitions;
661
+
662
+ // valid partition is the last valid partition in case threadid > num
663
+ // partitions
664
+ const int valid_partition =
665
+ (threadIdx.x < num_partitions) ? threadIdx.x : num_partitions - 1;
666
+ const int valid_partition2 = (WARP_SIZE + threadIdx.x < num_partitions)
667
+ ? WARP_SIZE + threadIdx.x
668
+ : num_partitions - 1;
669
+ float reg_max_logit = max_logits_ptr[valid_partition];
670
+ float reg_max_logit2 = max_logits_ptr[valid_partition2];
671
+ float max_logit = fmaxf(reg_max_logit, reg_max_logit2);
672
+
673
+ #pragma unroll
674
+ for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
675
+ max_logit = fmaxf(max_logit, __shfl_xor(max_logit, mask));
676
+ }
677
+
678
+ const float* exp_sums_ptr = exp_sums +
679
+ seq_idx * num_heads * max_num_partitions +
680
+ head_idx * max_num_partitions;
681
+
682
+ float global_exp_sum = 0.0f;
683
+ float rescaled_exp_sum = exp_sums_ptr[valid_partition];
684
+ float rescaled_exp_sum2 = exp_sums_ptr[valid_partition2];
685
+ rescaled_exp_sum *=
686
+ (threadIdx.x < num_partitions) ? expf(reg_max_logit - max_logit) : 0.0f;
687
+ rescaled_exp_sum2 *= (threadIdx.x + WARP_SIZE < num_partitions)
688
+ ? expf(reg_max_logit2 - max_logit)
689
+ : 0.0f;
690
+ global_exp_sum += rescaled_exp_sum + rescaled_exp_sum2;
691
+ shared_exp_sums[threadIdx.x] = rescaled_exp_sum;
692
+ shared_exp_sums[threadIdx.x + WARP_SIZE] = rescaled_exp_sum2;
693
+
694
+ #pragma unroll
695
+ for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
696
+ global_exp_sum += __shfl_xor(global_exp_sum, mask);
697
+ }
698
+ if (threadIdx.x == 0) {
699
+ shared_global_exp_sum = global_exp_sum;
700
+ }
701
+ } // warpid == 0
702
+ const scalar_t* tmp_out_ptr =
703
+ tmp_out + seq_idx * num_heads * max_num_partitions * HEAD_SIZE +
704
+ head_idx * max_num_partitions * HEAD_SIZE + threadIdx.x;
705
+ constexpr int MAX_NPAR = 64;
706
+ scalar_t tmps[MAX_NPAR];
707
+ const float dzero = 0.0f;
708
+ #pragma unroll
709
+ for (int j = 0; j < MAX_NPAR; j++) {
710
+ tmps[j] = from_float<scalar_t>(dzero);
711
+ }
712
+ const int last_partition_offset = (num_partitions - 1) * HEAD_SIZE;
713
+ const int num_partition_offset = (num_partitions)*HEAD_SIZE;
714
+ int idx = 0;
715
+
716
+ constexpr int JCHUNK = 16;
717
+
718
+ #pragma unroll
719
+ for (int j = 0; j < JCHUNK * HEAD_SIZE; j += HEAD_SIZE) {
720
+ // lastj is last valid partition
721
+ const int lastj_offset =
722
+ (j < num_partition_offset) ? j : last_partition_offset;
723
+ tmps[idx] = tmp_out_ptr[lastj_offset];
724
+ idx++;
725
+ }
726
+ __syncthreads();
727
+
728
+ if (num_partitions > JCHUNK) {
729
+ #pragma unroll
730
+ for (int j = JCHUNK * HEAD_SIZE; j < 2 * JCHUNK * HEAD_SIZE;
731
+ j += HEAD_SIZE) {
732
+ const int lastj_offset =
733
+ (j < num_partition_offset) ? j : last_partition_offset;
734
+ tmps[idx] = tmp_out_ptr[lastj_offset];
735
+ idx++;
736
+ }
737
+
738
+ if (num_partitions > 2 * JCHUNK) {
739
+ #pragma unroll
740
+ for (int j = 2 * JCHUNK * HEAD_SIZE; j < MAX_NPAR * HEAD_SIZE;
741
+ j += HEAD_SIZE) {
742
+ const int lastj_offset =
743
+ (j < num_partition_offset) ? j : last_partition_offset;
744
+ tmps[idx] = tmp_out_ptr[lastj_offset];
745
+ idx++;
746
+ }
747
+ }
748
+ } // num_partitions > JCHUNK
749
+
750
+ // Aggregate tmp_out to out.
751
+ float acc = 0.0f;
752
+ #pragma unroll
753
+ for (int j = 0; j < JCHUNK; j++) {
754
+ acc += to_float<scalar_t>(tmps[j]) * shared_exp_sums[j];
755
+ }
756
+ if (num_partitions > JCHUNK) {
757
+ #pragma unroll
758
+ for (int j = JCHUNK; j < 2 * JCHUNK; j++) {
759
+ acc += to_float<scalar_t>(tmps[j]) * shared_exp_sums[j];
760
+ }
761
+ if (num_partitions > 2 * JCHUNK) {
762
+ #pragma unroll
763
+ for (int j = 2 * JCHUNK; j < MAX_NPAR; j++) {
764
+ acc += to_float<scalar_t>(tmps[j]) * shared_exp_sums[j];
765
+ }
766
+ }
767
+ }
768
+
769
+ if (num_partitions > MAX_NPAR) {
770
+ idx = 0;
771
+ #pragma unroll
772
+ for (int j = MAX_NPAR * HEAD_SIZE; j < 2 * MAX_NPAR * HEAD_SIZE;
773
+ j += HEAD_SIZE) {
774
+ // lastj is last valid partition
775
+ const int lastj_offset =
776
+ (j < num_partition_offset) ? j : last_partition_offset;
777
+ tmps[idx] = tmp_out_ptr[lastj_offset];
778
+ idx++;
779
+ }
780
+
781
+ #pragma unroll
782
+ for (int j = 0; j < MAX_NPAR; j++) {
783
+ acc += to_float<scalar_t>(tmps[j]) * shared_exp_sums[j + MAX_NPAR];
784
+ }
785
+ }
786
+
787
+ const float inv_global_exp_sum =
788
+ __fdividef(1.0f, shared_global_exp_sum + 1e-6f);
789
+ acc *= inv_global_exp_sum;
790
+ scalar_t* out_ptr =
791
+ out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
792
+ out_ptr[threadIdx.x] = from_float<scalar_t>(acc);
793
+}
794
+
795
+#else // !defined(__HIP__MI300_MI250__) TODO: Add NAVI support
796
+
797
+template <typename scalar_t, int BLOCK_SIZE, int HEAD_SIZE, int NUM_THREADS,
798
+ int GQA_RATIO>
799
+__global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
800
+ const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
801
+ const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
802
+ // head_size/x, block_size, x]
803
+ const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
804
+ // head_size, block_size]
805
+ const int num_kv_heads, const float scale,
806
+ const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
807
+ const int* __restrict__ context_lens, // [num_seqs]
808
+ const int max_num_blocks_per_seq,
809
+ const float* __restrict__ alibi_slopes, // [num_heads]
810
+ const int q_stride, const int kv_block_stride, const int kv_head_stride,
811
+ float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
812
+ float* __restrict__ max_logits, // [num_seqs, num_heads,
813
+ // max_num_partitions]
814
+ scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions,
815
+ // head_size]
816
+ scalar_t* __restrict__ final_out, // [num_seqs, num_heads, head_size]
817
+ #if 0
818
+ scalar_t* __restrict__ qk_out, // [num_heads, num_seqs, max_ctx_blocks,block_size]
819
+ #endif
820
+ int max_ctx_blocks) {
821
+ UNREACHABLE_CODE
822
+}
823
+
824
+// Grid: (num_heads, num_seqs).
825
+template <typename scalar_t, int HEAD_SIZE, int NUM_THREADS,
826
+ int PARTITION_SIZE>
827
+__global__
828
+__launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
829
+ scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
830
+ const float* __restrict__ exp_sums, // [num_seqs, num_heads,
831
+ // max_num_partitions]
832
+ const float* __restrict__ max_logits, // [num_seqs, num_heads,
833
+ // max_num_partitions]
834
+ const scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads,
835
+ // max_num_partitions, head_size]
836
+ const int* __restrict__ context_lens, // [num_seqs]
837
+ const int max_num_partitions){UNREACHABLE_CODE}
838
+
839
+#endif // defined(__HIP__MI300_MI250__) TODO: Add NAVI support
840
+
841
+#define LAUNCH_CUSTOM_ATTENTION(GQA_RATIO) \
842
+ paged_attention_ll4mi_QKV_kernel<T, BLOCK_SIZE, HEAD_SIZE, NTHR, GQA_RATIO> \
843
+ <<<grid, block, 0, stream>>>( \
844
+ query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, scale, \
845
+ block_tables_ptr, context_lens_ptr, max_num_blocks_per_seq, \
846
+ alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
847
+ exp_sums_ptr, max_logits_ptr, tmp_out_ptr, out_ptr, max_ctx_blocks);
848
+
849
+template <typename T, int BLOCK_SIZE, int HEAD_SIZE, int PARTITION_SIZE = 256>
850
+void paged_attention_custom_launcher(
851
+ torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
852
+ torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
853
+ torch::Tensor& value_cache, const int num_kv_heads, float scale,
854
+ torch::Tensor& block_tables, torch::Tensor& context_lens,
855
+ int max_context_len,
856
+#if 0
857
+ torch::Tensor& qk_out,
858
+ torch::Tensor& softmax_out,
859
+#endif
860
+ const c10::optional<torch::Tensor>& alibi_slopes) {
861
+
862
+ int num_seqs = query.size(0);
863
+ int num_heads = query.size(1);
864
+ int head_size = query.size(2);
865
+ int max_num_blocks_per_seq = block_tables.size(1);
866
+ int q_stride = query.stride(0);
867
+ int kv_block_stride = key_cache.stride(0);
868
+ int kv_head_stride = key_cache.stride(1);
869
+
870
+ // NOTE: alibi_slopes is optional.
871
+ const float* alibi_slopes_ptr =
872
+ alibi_slopes
873
+ ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
874
+ : nullptr;
875
+
876
+ T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
877
+ float* exp_sums_ptr = reinterpret_cast<float*>(exp_sums.data_ptr());
878
+ float* max_logits_ptr = reinterpret_cast<float*>(max_logits.data_ptr());
879
+ T* tmp_out_ptr = reinterpret_cast<T*>(tmp_out.data_ptr());
880
+ T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
881
+ T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
882
+ T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
883
+ int* block_tables_ptr = block_tables.data_ptr<int>();
884
+ int* context_lens_ptr = context_lens.data_ptr<int>();
885
+#if 0
886
+ T* qk_out_ptr = reinterpret_cast<T*>(qk_out.data_ptr());
887
+ T* softmax_out_ptr = reinterpret_cast<T*>(softmax_out.data_ptr());
888
+#endif
889
+
890
+ const int max_ctx_blocks = DIVIDE_ROUND_UP(max_context_len, BLOCK_SIZE);
891
+ const int max_num_partitions =
892
+ DIVIDE_ROUND_UP(max_context_len, PARTITION_SIZE);
893
+ const int gqa_ratio = num_heads / num_kv_heads;
894
+ assert(num_heads % num_kv_heads == 0);
895
+ assert(head_size == HEAD_SIZE);
896
+ assert(max_num_partitions <= 128);
897
+
898
+ constexpr int NTHR = PARTITION_SIZE;
899
+ dim3 grid(num_seqs, max_num_partitions, num_kv_heads);
900
+ dim3 block(NTHR);
901
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
902
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
903
+ switch (gqa_ratio) {
904
+ case 1:
905
+ LAUNCH_CUSTOM_ATTENTION(1);
906
+ break;
907
+ case 2:
908
+ LAUNCH_CUSTOM_ATTENTION(2);
909
+ break;
910
+ case 3:
911
+ LAUNCH_CUSTOM_ATTENTION(3);
912
+ break;
913
+ case 4:
914
+ LAUNCH_CUSTOM_ATTENTION(4);
915
+ break;
916
+ case 5:
917
+ LAUNCH_CUSTOM_ATTENTION(5);
918
+ break;
919
+ case 6:
920
+ LAUNCH_CUSTOM_ATTENTION(6);
921
+ break;
922
+ case 7:
923
+ LAUNCH_CUSTOM_ATTENTION(7);
924
+ break;
925
+ case 8:
926
+ LAUNCH_CUSTOM_ATTENTION(8);
927
+ break;
928
+ case 9:
929
+ LAUNCH_CUSTOM_ATTENTION(9);
930
+ break;
931
+ case 10:
932
+ LAUNCH_CUSTOM_ATTENTION(10);
933
+ break;
934
+ case 11:
935
+ LAUNCH_CUSTOM_ATTENTION(11);
936
+ break;
937
+ case 12:
938
+ LAUNCH_CUSTOM_ATTENTION(12);
939
+ break;
940
+ case 13:
941
+ LAUNCH_CUSTOM_ATTENTION(13);
942
+ break;
943
+ case 14:
944
+ LAUNCH_CUSTOM_ATTENTION(14);
945
+ break;
946
+ case 15:
947
+ LAUNCH_CUSTOM_ATTENTION(15);
948
+ break;
949
+ case 16:
950
+ LAUNCH_CUSTOM_ATTENTION(16);
951
+ break;
952
+ default:
953
+ TORCH_CHECK(false, "Unsupported gqa ratio: ", gqa_ratio);
954
+ break;
955
+ }
956
+ // dim3 grid2(num_heads,num_seqs,head_size/HEAD_ELEMS_PER_WG);
957
+ // dim3 block2(1024);
958
+ // LAUNCH_CUSTOM_ATTENTION2;
959
+
960
+ // reduction kernel is only required if max_context_len > partition size,
961
+ // otherwise main kernel writes directly to final output
962
+ // note there are cases with graphing where max_context_len is the max
963
+ // supported by graphing, not the actual max among all the sequences: in that
964
+ // case reduction kernel will still run but return immediately
965
+ if (max_context_len > PARTITION_SIZE) {
966
+ dim3 reduce_grid(num_heads, num_seqs);
967
+ dim3 reduce_block(head_size);
968
+ paged_attention_ll4mi_reduce_kernel<T, HEAD_SIZE, HEAD_SIZE, PARTITION_SIZE>
969
+ <<<reduce_grid, reduce_block, 0, stream>>>(
970
+ out_ptr, exp_sums_ptr, max_logits_ptr, tmp_out_ptr,
971
+ context_lens_ptr, max_num_partitions);
972
+ }
973
+}
974
+
975
+#define CALL_CUSTOM_LAUNCHER(T, BLK_SIZE, HEAD_SIZE) \
976
+ paged_attention_custom_launcher<T, BLK_SIZE, HEAD_SIZE>( \
977
+ out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
978
+ num_kv_heads, scale, block_tables, context_lens, max_context_len, \
979
+ alibi_slopes);
980
+
981
+#define CALL_CUSTOM_LAUNCHER_BLK(T, HEAD_SIZE) \
982
+ switch (block_size) { \
983
+ case 16: \
984
+ CALL_CUSTOM_LAUNCHER(T, 16, HEAD_SIZE); \
985
+ break; \
986
+ case 32: \
987
+ CALL_CUSTOM_LAUNCHER(T, 32, HEAD_SIZE); \
988
+ break; \
989
+ default: \
990
+ TORCH_CHECK(false, "Unsupported block size: ", block_size); \
991
+ break; \
992
+ }
993
+
994
+#define CALL_CUSTOM_LAUNCHER_BLK_HEAD(T) \
995
+ switch (head_size) { \
996
+ case 64: \
997
+ CALL_CUSTOM_LAUNCHER_BLK(T, 64); \
998
+ break; \
999
+ case 128: \
1000
+ CALL_CUSTOM_LAUNCHER_BLK(T, 128); \
1001
+ break; \
1002
+ default: \
1003
+ TORCH_CHECK(false, "Unsupported head size: ", head_size); \
1004
+ break; \
1005
+ }
1006
+
1007
+void paged_attention(
1008
+ torch::Tensor& out, // [num_seqs, num_heads, head_size]
1009
+ torch::Tensor& exp_sums, // [num_seqs, num_heads, max_num_partitions]
1010
+ torch::Tensor& max_logits, // [num_seqs, num_heads, max_num_partitions]
1011
+ torch::Tensor&
1012
+ tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
1013
+ torch::Tensor& query, // [num_seqs, num_heads, head_size]
1014
+ torch::Tensor&
1015
+ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
1016
+ torch::Tensor&
1017
+ value_cache, // [num_blocks, num_heads, head_size, block_size]
1018
+ int64_t num_kv_heads, double scale,
1019
+ torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
1020
+ torch::Tensor& context_lens, // [num_seqs]
1021
+ int64_t block_size, int64_t max_context_len,
1022
+ const c10::optional<torch::Tensor>& alibi_slopes,
1023
+ const std::string& kv_cache_dtype) {
1024
+ assert(kv_cache_dtype == "auto");
1025
+ const int head_size = query.size(2);
1026
+ if (query.dtype() == at::ScalarType::Half) {
1027
+ CALL_CUSTOM_LAUNCHER_BLK_HEAD(_Float16);
1028
+ } else if (query.dtype() == at::ScalarType::BFloat16) {
1029
+ CALL_CUSTOM_LAUNCHER_BLK_HEAD(__hip_bfloat16);
1030
+ } else {
1031
+ TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
1032
+ }
1033
+}
1034
+
1035
+#undef WARP_SIZE
1036
+#undef MAX
1037
+#undef MIN
1038
+#undef DIVIDE_ROUND_UP
csrc/rocm/ops.h
new
+13
@@ -0,0 +1,13 @@
1
+#pragma once
2
+
3
+#include <torch/all.h>
4
+
5
+void paged_attention(torch::Tensor& out, torch::Tensor& exp_sums,
6
+ torch::Tensor& max_logits, torch::Tensor& tmp_out,
7
+ torch::Tensor& query, torch::Tensor& key_cache,
8
+ torch::Tensor& value_cache, int64_t num_kv_heads,
9
+ double scale, torch::Tensor& block_tables,
10
+ torch::Tensor& context_lens, int64_t block_size,
11
+ int64_t max_context_len,
12
+ const c10::optional<torch::Tensor>& alibi_slopes,
13
+ const std::string& kv_cache_dtype);
csrc/rocm/torch_bindings.cpp
new
+33
@@ -0,0 +1,33 @@
1
+#include "core/registration.h"
2
+#include "rocm/ops.h"
3
+
4
+// Note on op signatures:
5
+// The X_meta signatures are for the meta functions corresponding to op X.
6
+// They must be kept in sync with the signature for X. Generally, only
7
+// functions that return Tensors require a meta function.
8
+//
9
+// See the following links for detailed docs on op registration and function
10
+// schemas.
11
+// https://docs.google.com/document/d/1_W62p8WJOQQUzPsJYa7s701JXt0qf2OfLub2sbkHOaU/edit#heading=h.ptttacy8y1u9
12
+// https://github.com/pytorch/pytorch/blob/main/aten/src/ATen/native/README.md#annotations
13
+
14
+TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, rocm_ops) {
15
+ // vLLM custom ops for rocm
16
+
17
+ // Custom attention op
18
+ // Compute the attention between an input query and the cached
19
+ // keys/values using PagedAttention.
20
+ rocm_ops.def(
21
+ "paged_attention(Tensor! out, Tensor exp_sums,"
22
+ " Tensor max_logits, Tensor tmp_out,"
23
+ " Tensor query, Tensor key_cache,"
24
+ " Tensor value_cache, int num_kv_heads,"
25
+ " float scale, Tensor block_tables,"
26
+ " Tensor context_lens, int block_size,"
27
+ " int max_context_len,"
28
+ " Tensor? alibi_slopes,"
29
+ " str kv_cache_dtype) -> ()");
30
+ rocm_ops.impl("paged_attention", torch::kCUDA, &paged_attention);
31
+}
32
+
33
+REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
setup.py
+3
@@ -462,6 +462,9 @@ if _build_core_ext():
462
if _is_cuda() or _is_hip():
463
ext_modules.append(CMakeExtension(name="vllm._moe_C"))
464
465
+if _is_hip():
466
+ ext_modules.append(CMakeExtension(name="vllm._rocm_C"))
467
+
468
if _build_custom_ops():
469
ext_modules.append(CMakeExtension(name="vllm._C"))
470
tests/kernels/test_attention.py
+164
-2
@@ -3,8 +3,6 @@ from typing import List, Optional, Tuple
3
4
import pytest
5
import torch
6
-from xformers import ops as xops
7
-from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
6
7
from tests.kernels.utils import opcheck
8
from vllm import _custom_ops as ops
@@ -12,6 +10,10 @@ from vllm.utils import get_max_shared_memory_bytes, is_hip
10
11
from .allclose_default import get_default_atol, get_default_rtol
12
13
+if not is_hip():
14
+ from xformers import ops as xops
15
+ from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
16
+
17
FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
18
# This will change depending on the compute capability.
19
# - 512 as a buffer
@@ -328,6 +330,165 @@ def ref_multi_query_kv_attention(
330
return torch.cat(ref_outputs, dim=0)
331
332
333
+@pytest.mark.parametrize("version", ["rocm"])
334
+@pytest.mark.parametrize("num_seqs", NUM_GEN_SEQS)
335
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
336
+@pytest.mark.parametrize("head_size", [64, 128]) # only test 64 128
337
+@pytest.mark.parametrize("use_alibi", USE_ALIBI)
338
+@pytest.mark.parametrize("block_size", BLOCK_SIZES)
339
+@pytest.mark.parametrize("dtype", DTYPES)
340
+@pytest.mark.parametrize("kv_cache_dtype", ["auto"])
341
+@pytest.mark.parametrize("seed", SEEDS)
342
+@pytest.mark.parametrize("device", CUDA_DEVICES)
343
+@pytest.mark.skipif(not is_hip(), reason="only for rocm")
344
+def test_paged_attention_rocm(
345
+ kv_cache_factory,
346
+ version: str,
347
+ num_seqs: int,
348
+ num_heads: Tuple[int, int],
349
+ head_size: int,
350
+ use_alibi: bool,
351
+ block_size: int,
352
+ dtype: torch.dtype,
353
+ kv_cache_dtype: str,
354
+ seed: int,
355
+ device: str,
356
+) -> None:
357
+ random.seed(seed)
358
+ torch.random.manual_seed(seed)
359
+ if torch.cuda.is_available():
360
+ torch.cuda.manual_seed(seed)
361
+ torch.set_default_device(device)
362
+ scale = float(1.0 / (head_size**0.5))
363
+ num_query_heads, num_kv_heads = num_heads
364
+ query = torch.empty(num_seqs, num_query_heads, head_size, dtype=dtype)
365
+ query.uniform_(-scale, scale)
366
+
367
+ assert num_query_heads % num_kv_heads == 0
368
+ num_queries_per_kv = num_query_heads // num_kv_heads
369
+ alibi_slopes = None
370
+ if use_alibi:
371
+ alibi_slopes = torch.randn(num_query_heads, dtype=torch.float)
372
+
373
+ context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
374
+ context_lens[-1] = MAX_SEQ_LEN
375
+ #context_lens = [8192 for _ in range(num_seqs)]
376
+ max_context_len = max(context_lens)
377
+ context_lens = torch.tensor(context_lens, dtype=torch.int)
378
+ #print('>>> ctx lens', context_lens)
379
+
380
+ # Create the block tables.
381
+ max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
382
+ block_tables = []
383
+ for _ in range(num_seqs):
384
+ block_table = [
385
+ random.randint(0, NUM_BLOCKS - 1)
386
+ for _ in range(max_num_blocks_per_seq)
387
+ ]
388
+ block_tables.append(block_table)
389
+ block_tables = torch.tensor(block_tables, dtype=torch.int)
390
+
391
+ # Create the KV caches.
392
+ key_caches, value_caches = kv_cache_factory(NUM_BLOCKS, block_size, 1,
393
+ num_kv_heads, head_size,
394
+ kv_cache_dtype, dtype, seed,
395
+ device)
396
+ key_cache, value_cache = key_caches[0], value_caches[0]
397
+
398
+ # TODO(charlifu) enable fp8 kv cache
399
+ # Using default kv_scale
400
+ # kv_scale = 1.0
401
+
402
+ # Call the paged attention kernel.
403
+ output = torch.empty_like(query)
404
+ PARTITION_SIZE_ROCM = 256
405
+ num_partitions = ((max_context_len + PARTITION_SIZE_ROCM - 1) //
406
+ PARTITION_SIZE_ROCM)
407
+ assert PARTITION_SIZE_ROCM % block_size == 0
408
+ num_seqs, num_heads, head_size = output.shape
409
+ tmp_output = torch.empty(
410
+ size=(num_seqs, num_heads, num_partitions, head_size),
411
+ dtype=output.dtype,
412
+ )
413
+ exp_sums = torch.empty(
414
+ size=(num_seqs, num_heads, num_partitions),
415
+ dtype=torch.float32,
416
+ )
417
+ max_logits = torch.empty_like(exp_sums)
418
+ if version == "rocm":
419
+ ops.paged_attention_rocm(
420
+ output,
421
+ exp_sums,
422
+ max_logits,
423
+ tmp_output,
424
+ query,
425
+ key_cache,
426
+ value_cache,
427
+ num_kv_heads,
428
+ scale,
429
+ block_tables,
430
+ context_lens,
431
+ block_size,
432
+ max_context_len,
433
+ alibi_slopes,
434
+ kv_cache_dtype,
435
+ )
436
+ else:
437
+ raise AssertionError(f"Unknown version: {version}")
438
+
439
+ # Run the reference implementation.
440
+ if kv_cache_dtype == "fp8":
441
+ # Convert cache data back to dtype.
442
+ x = 16 // torch.tensor([], dtype=dtype).element_size()
443
+ key_cache_shape = (NUM_BLOCKS, num_kv_heads, head_size // x,
444
+ block_size, x)
445
+ dequantized_key_cache = torch.empty(size=key_cache_shape,
446
+ dtype=dtype,
447
+ device=device)
448
+ ops.convert_fp8(key_cache, dequantized_key_cache)
449
+ key_cache = dequantized_key_cache
450
+
451
+ value_cache_shape = value_cache.shape
452
+ dequantized_value_cache = torch.empty(size=value_cache_shape,
453
+ dtype=dtype,
454
+ device=device)
455
+ ops.convert_fp8(value_cache, dequantized_value_cache)
456
+ value_cache = dequantized_value_cache
457
+
458
+ ref_output = torch.empty_like(query)
459
+ ref_single_query_cached_kv_attention(
460
+ ref_output,
461
+ query,
462
+ num_queries_per_kv,
463
+ key_cache,
464
+ value_cache,
465
+ block_tables,
466
+ context_lens,
467
+ scale,
468
+ alibi_slopes,
469
+ )
470
+
471
+ # NOTE(woosuk): Due to the kernel-level differences in the two
472
+ # implementations, there is a small numerical difference in the two
473
+ # outputs. Thus, we use a relaxed tolerance for the test.
474
+ atol = get_default_atol(output) if is_hip() else 1e-3
475
+ rtol = get_default_rtol(output) if is_hip() else 1e-5
476
+
477
+ # NOTE(zhaoyang): FP8 KV Cache will introduce quantization error,
478
+ # so we use a relaxed tolerance for the test.
479
+ atol, rtol = 1e-4, 1e-5
480
+ if dtype == torch.bfloat16:
481
+ atol, rtol = 2e-4, 1e-5
482
+ if use_alibi:
483
+ if dtype == torch.half:
484
+ atol, rtol = 5e-4, 1e-5
485
+ if dtype == torch.bfloat16:
486
+ atol, rtol = 1e-3, 1e-5
487
+ if kv_cache_dtype == "fp8":
488
+ atol, rtol = 1e-2, 1e-5
489
+ assert torch.allclose(output, ref_output, atol=atol, rtol=rtol)
490
+
491
+
492
# TODO(woosuk): Add tests for USE_ALIBI=True.
493
@pytest.mark.parametrize("num_seqs", NUM_PREFILL_SEQS)
494
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@@ -335,6 +496,7 @@ def ref_multi_query_kv_attention(
496
@pytest.mark.parametrize("dtype", DTYPES)
497
@pytest.mark.parametrize("seed", SEEDS)
498
@pytest.mark.parametrize("device", CUDA_DEVICES)
499
+@pytest.mark.skipif(is_hip(), reason="skip for rocm")
500
@torch.inference_mode()
501
def test_multi_query_kv_attention(
502
num_seqs: int,
vllm/_custom_ops.py
+27
@@ -17,6 +17,9 @@ if not current_platform.is_tpu():
17
except ImportError as e:
18
logger.warning("Failed to import from vllm._C with %r", e)
19
20
+if current_platform.is_rocm():
21
+ import vllm._rocm_C # noqa: F401
22
+
23
with contextlib.suppress(ImportError):
24
import vllm._moe_C # noqa: F401
25
@@ -127,6 +130,30 @@ def paged_attention_v2(
130
blocksparse_block_size, blocksparse_head_sliding_step)
131
132
133
+def paged_attention_rocm(
134
+ out: torch.Tensor,
135
+ exp_sum: torch.Tensor,
136
+ max_logits: torch.Tensor,
137
+ tmp_out: torch.Tensor,
138
+ query: torch.Tensor,
139
+ key_cache: torch.Tensor,
140
+ value_cache: torch.Tensor,
141
+ num_kv_heads: int,
142
+ scale: float,
143
+ block_tables: torch.Tensor,
144
+ seq_lens: torch.Tensor,
145
+ block_size: int,
146
+ max_seq_len: int,
147
+ alibi_slopes: Optional[torch.Tensor],
148
+ kv_cache_dtype: str,
149
+) -> None:
150
+ torch.ops._rocm_C.paged_attention(out, exp_sum, max_logits, tmp_out, query,
151
+ key_cache, value_cache, num_kv_heads,
152
+ scale, block_tables, seq_lens,
153
+ block_size, max_seq_len, alibi_slopes,
154
+ kv_cache_dtype)
155
+
156
+
157
# pos encoding ops
158
def rotary_embedding(
159
positions: torch.Tensor,
vllm/attention/backends/rocm_flash_attn.py
+70
-14
@@ -5,6 +5,7 @@ from typing import Any, Dict, List, Optional, Tuple, Type
5
import torch
6
7
import vllm.envs as envs
8
+from vllm import _custom_ops as ops
9
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
AttentionMetadata, AttentionType)
11
from vllm.attention.backends.utils import (CommonAttentionState,
@@ -15,6 +16,9 @@ from vllm.logger import init_logger
16
17
logger = init_logger(__name__)
18
19
+_PARTITION_SIZE = 256
20
+ON_NAVI = "gfx1" in torch.cuda.get_device_properties("cuda").gcnArchName
21
+
22
23
class ROCmFlashAttentionBackend(AttentionBackend):
24
@@ -480,20 +484,61 @@ class ROCmFlashAttentionImpl(AttentionImpl):
484
485
if decode_meta := attn_metadata.decode_metadata:
486
# Decoding run.
483
- output[num_prefill_tokens:] = PagedAttention.forward_decode(
484
- decode_query,
485
- key_cache,
486
- value_cache,
487
- decode_meta.block_tables,
488
- decode_meta.seq_lens_tensor,
489
- decode_meta.max_decode_seq_len,
490
- self.kv_cache_dtype,
491
- self.num_kv_heads,
492
- self.scale,
493
- self.alibi_slopes,
494
- k_scale,
495
- v_scale,
496
- )
487
+ # Whether to use rocm custom paged attention or not
488
+ num_seqs, num_heads, head_size = decode_query.shape
489
+ block_size = value_cache.shape[3]
490
+ gqa_ratio = num_heads // self.num_kv_heads
491
+ use_custom = use_rocm_custom_paged_attention(
492
+ decode_query.dtype, head_size, block_size, self.kv_cache_dtype,
493
+ gqa_ratio, decode_meta.max_decode_seq_len)
494
+ if use_custom:
495
+ max_seq_len = decode_meta.max_decode_seq_len
496
+ max_num_partitions = ((max_seq_len + _PARTITION_SIZE - 1) //
497
+ _PARTITION_SIZE)
498
+ assert _PARTITION_SIZE % block_size == 0
499
+ tmp_output = torch.empty(
500
+ size=(num_seqs, num_heads, max_num_partitions, head_size),
501
+ dtype=output.dtype,
502
+ device=output.device,
503
+ )
504
+ exp_sums = torch.empty(
505
+ size=(num_seqs, num_heads, max_num_partitions),
506
+ dtype=torch.float32,
507
+ device=output.device,
508
+ )
509
+ max_logits = torch.empty_like(exp_sums)
510
+ ops.paged_attention_rocm(
511
+ output[num_prefill_tokens:],
512
+ exp_sums,
513
+ max_logits,
514
+ tmp_output,
515
+ decode_query,
516
+ key_cache,
517
+ value_cache,
518
+ self.num_kv_heads,
519
+ self.scale,
520
+ decode_meta.block_tables,
521
+ decode_meta.seq_lens_tensor,
522
+ block_size,
523
+ max_seq_len,
524
+ self.alibi_slopes,
525
+ self.kv_cache_dtype,
526
+ )
527
+ else:
528
+ output[num_prefill_tokens:] = PagedAttention.forward_decode(
529
+ decode_query,
530
+ key_cache,
531
+ value_cache,
532
+ decode_meta.block_tables,
533
+ decode_meta.seq_lens_tensor,
534
+ decode_meta.max_decode_seq_len,
535
+ self.kv_cache_dtype,
536
+ self.num_kv_heads,
537
+ self.scale,
538
+ self.alibi_slopes,
539
+ k_scale,
540
+ v_scale,
541
+ )
542
543
# Reshape the output tensor.
544
return output.view(num_tokens, hidden_size)
@@ -532,3 +577,14 @@ def _sdpa_attention(
577
start = end
578
579
return output
580
+
581
+
582
+def use_rocm_custom_paged_attention(qtype: torch.dtype, head_size: int,
583
+ block_size: int, kv_cache_dtype: str,
584
+ gqa_ratio: int, max_seq_len: int) -> bool:
585
+ # rocm custom page attention not support on navi (gfx1*)
586
+ return (not ON_NAVI and (qtype == torch.half or qtype == torch.bfloat16)
587
+ and (head_size == 64 or head_size == 128)
588
+ and (block_size == 16 or block_size == 32)
589
+ and kv_cache_dtype == "auto"
590
+ and (gqa_ratio >= 1 and gqa_ratio <= 16) and max_seq_len <= 32768)