A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Kernel][Amd] Add fp8 kv cache support for rocm custom paged attention (#8577)
Charlie Fu committed
Sep 19, 2024 at 12:37 UTC
9cc373f39036af789fb1ffc1e06b23766996d3f4
6 files changed
+246
-283
csrc/rocm/attention.cu
+161
-79
@@ -18,8 +18,11 @@
18
#include <ATen/cuda/CUDAContext.h>
19
#include <c10/cuda/CUDAGuard.h>
20
#include <hip/hip_bf16.h>
21
+#include "cuda_compat.h"
22
23
#include <algorithm>
24
+#include "../attention/dtype_fp8.cuh"
25
+#include "../quantization/fp8/amd/quant_utils.cuh"
26
27
#if defined(__HIPCC__) && (defined(__gfx90a__) || defined(__gfx940__) || \
28
defined(__gfx941__) || defined(__gfx942__))
@@ -38,7 +41,6 @@
41
#define MAX(a, b) ((a) > (b) ? (a) : (b))
42
#define MIN(a, b) ((a) < (b) ? (a) : (b))
43
#define DIVIDE_ROUND_UP(a, b) (((a) + (b) - 1) / (b))
41
-#define WARP_SIZE 64
44
45
#if defined(__HIP__MI300_MI250__) // TODO: Add NAVI support
46
@@ -60,6 +62,8 @@ typedef struct _B16x8 {
62
_B16x4 xy[2];
63
} _B16x8;
64
65
+using _B8x8 = uint2;
66
+
67
////// Non temporal load stores ///////
68
69
template <typename T>
@@ -168,18 +172,40 @@ __device__ __forceinline__ _B16x4 addx4(const _B16x4& inp1,
172
}
173
}
174
175
+template <typename T, vllm::Fp8KVCacheDataType KV_DTYPE>
176
+__device__ __forceinline__ _B16x8 scaled_convert_b8x8(const _B8x8 input,
177
+ const float scale) {
178
+ union alignas(16) {
179
+ uint4 u4;
180
+ _B16x8 u16x8;
181
+ vllm::bf16_8_t b16x8;
182
+ } tmp;
183
+ if constexpr (std::is_same<T, _Float16>::value) {
184
+ tmp.u4 = vllm::fp8::scaled_convert<uint4, _B8x8, KV_DTYPE>(input, scale);
185
+ return tmp.u16x8;
186
+ } else if constexpr (std::is_same<T, __hip_bfloat16>::value) {
187
+ tmp.b16x8 = vllm::fp8::scaled_convert<vllm::bf16_8_t, _B8x8, KV_DTYPE>(
188
+ input, scale);
189
+ return tmp.u16x8;
190
+ } else {
191
+ static_assert(false, "unsupported 16b dtype");
192
+ }
193
+}
194
+
195
///////////////////////////////////////
196
197
// grid (num_seqs, num_partitions,num_heads/gqa_ratio)
198
// block (partition size)
175
-template <typename scalar_t, int BLOCK_SIZE, int HEAD_SIZE, int NUM_THREADS,
199
+template <typename scalar_t, typename cache_t,
200
+ vllm::Fp8KVCacheDataType KV_DTYPE, int BLOCK_SIZE, int HEAD_SIZE,
201
+ int NUM_THREADS,
202
int GQA_RATIO>
203
__global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
178
- const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
179
- const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
180
- // head_size/x, block_size, x]
181
- const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
182
- // head_size, block_size]
204
+ const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
205
+ const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
206
+ // head_size/x, block_size, x]
207
+ const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
208
+ // head_size, block_size]
209
const int num_kv_heads, const float scale,
210
const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
211
const int* __restrict__ context_lens, // [num_seqs]
@@ -192,10 +218,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
218
scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions,
219
// head_size]
220
scalar_t* __restrict__ final_out, // [num_seqs, num_heads, head_size]
195
- #if 0
196
- scalar_t* __restrict__ qk_out, // [num_heads, num_seqs, max_ctx_blocks,block_size]
197
- #endif
198
- int max_ctx_blocks) {
221
+ int max_ctx_blocks, float k_scale, float v_scale) {
222
constexpr int NWARPS = NUM_THREADS / WARP_SIZE;
223
const int warpid = threadIdx.x / WARP_SIZE;
224
const int laneid = threadIdx.x % WARP_SIZE;
@@ -222,12 +245,14 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
245
constexpr int x = 16 / sizeof(scalar_t);
246
constexpr int KHELOOP = HEAD_SIZE / x;
247
_B16x8 Klocal[KHELOOP];
248
+ _B8x8 Klocalb8[KHELOOP];
249
constexpr int VHELOOP =
250
HEAD_SIZE /
251
WARP_SIZE; // v head_size dimension is distributed across lanes
252
constexpr int VTLOOP = 8; // 16 separate 4xtokens across warp -> 16/2
253
// 8xtokens
254
_B16x8 Vlocal[VHELOOP][VTLOOP];
255
+ _B8x8 Vlocalb8[VHELOOP][VTLOOP];
256
floatx4 dout[QHLOOP];
257
float qk_max[QHLOOP];
258
#pragma unroll
@@ -279,6 +304,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
304
(vblock_idx <= last_ctx_block) ? vblock_idx : last_ctx_block;
305
vphysical_blocks[b] = block_table[vblock_idx_ctx];
306
}
307
+
308
// each 4 lanes fetch 8 helems, so warp fetches 8*16 = 128 helems
309
const scalar_t* q_ptr =
310
q + seq_idx * q_stride + wg_start_head_idx * HEAD_SIZE;
@@ -298,17 +324,29 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
324
Qlocal[QHLOOP - 1].xy[1] = {0};
325
}
326
301
- const scalar_t* k_ptr = k_cache + physical_block_number * kv_block_stride +
302
- wg_start_kv_head_idx * kv_head_stride;
327
+ const cache_t* k_ptr = k_cache + physical_block_number * kv_block_stride +
328
+ wg_start_kv_head_idx * kv_head_stride;
329
330
const int physical_block_offset =
331
local_token_idx % BLOCK_SIZE; // since x=half8, physical_block_offset
332
// is already cast as _H8
307
-
308
- const _B16x8* k_ptrh8 = reinterpret_cast<const _B16x8*>(k_ptr);
333
+ if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) {
334
+ const _B16x8* k_ptrh8 = reinterpret_cast<const _B16x8*>(k_ptr);
335
+ #pragma unroll
336
+ for (int d = 0; d < KHELOOP; d++) {
337
+ Klocal[d] = k_ptrh8[d * BLOCK_SIZE + physical_block_offset];
338
+ }
339
+ } else {
340
+ constexpr int X = 16 / sizeof(cache_t);
341
+ const cache_t* k_ptr2 = k_ptr + physical_block_offset * X;
342
#pragma unroll
310
- for (int d = 0; d < KHELOOP; d++) {
311
- Klocal[d] = k_ptrh8[d * BLOCK_SIZE + physical_block_offset];
343
+ for (int d = 0; d < KHELOOP; d++) {
344
+ const int head_elem = d * 8;
345
+ const int offset1 = head_elem / X;
346
+ const int offset2 = head_elem % X;
347
+ const cache_t* k_ptr3 = k_ptr2 + offset1 * BLOCK_SIZE * X + offset2;
348
+ Klocalb8[d] = *reinterpret_cast<const _B8x8*>(k_ptr3);
349
+ }
350
}
351
352
float alibi_slope[QHLOOP];
@@ -322,30 +360,66 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
360
}
361
}
362
325
- const scalar_t* v_ptr = v_cache + wg_start_kv_head_idx * kv_head_stride;
326
- const _B16x8* v_ptrh8 = reinterpret_cast<const _B16x8*>(v_ptr);
327
- // iterate over each v block
363
+ const cache_t* v_ptr = v_cache + wg_start_kv_head_idx * kv_head_stride;
364
+ if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) {
365
+ const _B16x8* v_ptrh8 = reinterpret_cast<const _B16x8*>(v_ptr);
366
+ // iterate over each v block
367
#pragma unroll
329
- for (int b = 0; b < VBLOCKS; b++) {
330
- // int32 physical_block_number leads to overflow when multiplied with
331
- // kv_block_stride
332
- const int64_t vphysical_block_number =
333
- static_cast<int64_t>(vphysical_blocks[b]);
334
- const _B16x8* v_ptrh8b =
335
- v_ptrh8 + (vphysical_block_number * kv_block_stride) / 8;
336
- // iterate over each head elem (within head_size)
368
+ for (int b = 0; b < VBLOCKS; b++) {
369
+ // int32 physical_block_number leads to overflow when multiplied with
370
+ // kv_block_stride
371
+ const int64_t vphysical_block_number =
372
+ static_cast<int64_t>(vphysical_blocks[b]);
373
+ const _B16x8* v_ptrh8b =
374
+ v_ptrh8 + (vphysical_block_number * kv_block_stride) / 8;
375
+ // iterate over each head elem (within head_size)
376
+ #pragma unroll
377
+ for (int h = 0; h < VHELOOP; h++) {
378
+ const int head_size_elem = h * WARP_SIZE + laneid;
379
+ const _B16x8* v_ptrh8be = v_ptrh8b + head_size_elem * BLOCK_SIZE / 8;
380
+ // iterate over all velems within block
381
+ #pragma unroll
382
+ for (int d = 0; d < BLOCK_SIZE / 8; d++) {
383
+ Vlocal[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
384
+ }
385
+ }
386
+ }
387
+ } else {
388
+ const _B8x8* v_ptrh8 = reinterpret_cast<const _B8x8*>(v_ptr);
389
+ // iterate over each v block
390
+ #pragma unroll
391
+ for (int b = 0; b < VBLOCKS; b++) {
392
+ // int32 physical_block_number leads to overflow when multiplied with
393
+ // kv_block_stride
394
+ const int64_t vphysical_block_number =
395
+ static_cast<int64_t>(vphysical_blocks[b]);
396
+ const _B8x8* v_ptrh8b =
397
+ v_ptrh8 + (vphysical_block_number * kv_block_stride) / 8;
398
+ // iterate over each head elem (within head_size)
399
#pragma unroll
338
- for (int h = 0; h < VHELOOP; h++) {
339
- const int head_size_elem = h * WARP_SIZE + laneid;
340
- const _B16x8* v_ptrh8be = v_ptrh8b + head_size_elem * BLOCK_SIZE / 8;
341
- // iterate over all velems within block
400
+ for (int h = 0; h < VHELOOP; h++) {
401
+ const int head_size_elem = h * WARP_SIZE + laneid;
402
+ const _B8x8* v_ptrh8be = v_ptrh8b + head_size_elem * BLOCK_SIZE / 8;
403
+ // iterate over all velems within block
404
#pragma unroll
343
- for (int d = 0; d < BLOCK_SIZE / 8; d++) {
344
- Vlocal[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
405
+ for (int d = 0; d < BLOCK_SIZE / 8; d++) {
406
+ // Vlocalb8[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
407
+ const _B8x8 Vlocalb8 = v_ptrh8be[d];
408
+ Vlocal[h][b * BLOCK_SIZE / 8 + d] =
409
+ scaled_convert_b8x8<scalar_t, KV_DTYPE>(Vlocalb8, v_scale);
410
+ }
411
}
412
}
413
}
414
415
+ if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) {
416
+ #pragma unroll
417
+ for (int d = 0; d < KHELOOP; d++) {
418
+ Klocal[d] =
419
+ scaled_convert_b8x8<scalar_t, KV_DTYPE>(Klocalb8[d], k_scale);
420
+ }
421
+ }
422
+
423
#pragma unroll
424
for (int h = 0; h < QHLOOP; h++) {
425
dout[h] = gcn_mfma_instr<scalar_t, 4, 0, 0>(Qlocal[h].xy[0],
@@ -794,14 +868,16 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
868
869
#else // !defined(__HIP__MI300_MI250__) TODO: Add NAVI support
870
797
-template <typename scalar_t, int BLOCK_SIZE, int HEAD_SIZE, int NUM_THREADS,
871
+template <typename scalar_t, typename cache_t,
872
+ vllm::Fp8KVCacheDataType KV_DTYPE, int BLOCK_SIZE, int HEAD_SIZE,
873
+ int NUM_THREADS,
874
int GQA_RATIO>
875
__global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
800
- const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
801
- const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
802
- // head_size/x, block_size, x]
803
- const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
804
- // head_size, block_size]
876
+ const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
877
+ const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
878
+ // head_size/x, block_size, x]
879
+ const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
880
+ // head_size, block_size]
881
const int num_kv_heads, const float scale,
882
const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
883
const int* __restrict__ context_lens, // [num_seqs]
@@ -814,10 +890,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
890
scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions,
891
// head_size]
892
scalar_t* __restrict__ final_out, // [num_seqs, num_heads, head_size]
817
- #if 0
818
- scalar_t* __restrict__ qk_out, // [num_heads, num_seqs, max_ctx_blocks,block_size]
819
- #endif
820
- int max_ctx_blocks) {
893
+ int max_ctx_blocks, float k_scale, float v_scale) {
894
UNREACHABLE_CODE
895
}
896
@@ -839,26 +912,24 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
912
#endif // defined(__HIP__MI300_MI250__) TODO: Add NAVI support
913
914
#define LAUNCH_CUSTOM_ATTENTION(GQA_RATIO) \
842
- paged_attention_ll4mi_QKV_kernel<T, BLOCK_SIZE, HEAD_SIZE, NTHR, GQA_RATIO> \
915
+ paged_attention_ll4mi_QKV_kernel<T, KVT, KV_DTYPE, BLOCK_SIZE, HEAD_SIZE, \
916
+ NTHR, GQA_RATIO> \
917
<<<grid, block, 0, stream>>>( \
918
query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, scale, \
919
block_tables_ptr, context_lens_ptr, max_num_blocks_per_seq, \
920
alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
847
- exp_sums_ptr, max_logits_ptr, tmp_out_ptr, out_ptr, max_ctx_blocks);
921
+ exp_sums_ptr, max_logits_ptr, tmp_out_ptr, out_ptr, max_ctx_blocks, \
922
+ k_scale, v_scale);
923
849
-template <typename T, int BLOCK_SIZE, int HEAD_SIZE, int PARTITION_SIZE = 256>
924
+template <typename T, typename KVT, vllm::Fp8KVCacheDataType KV_DTYPE,
925
+ int BLOCK_SIZE, int HEAD_SIZE, int PARTITION_SIZE = 512>
926
void paged_attention_custom_launcher(
927
torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
928
torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
929
torch::Tensor& value_cache, const int num_kv_heads, float scale,
930
torch::Tensor& block_tables, torch::Tensor& context_lens,
855
- int max_context_len,
856
-#if 0
857
- torch::Tensor& qk_out,
858
- torch::Tensor& softmax_out,
859
-#endif
860
- const c10::optional<torch::Tensor>& alibi_slopes) {
861
-
931
+ int max_context_len, const c10::optional<torch::Tensor>& alibi_slopes,
932
+ float k_scale, float v_scale) {
933
int num_seqs = query.size(0);
934
int num_heads = query.size(1);
935
int head_size = query.size(2);
@@ -878,14 +949,10 @@ void paged_attention_custom_launcher(
949
float* max_logits_ptr = reinterpret_cast<float*>(max_logits.data_ptr());
950
T* tmp_out_ptr = reinterpret_cast<T*>(tmp_out.data_ptr());
951
T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
881
- T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
882
- T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
952
+ KVT* key_cache_ptr = reinterpret_cast<KVT*>(key_cache.data_ptr());
953
+ KVT* value_cache_ptr = reinterpret_cast<KVT*>(value_cache.data_ptr());
954
int* block_tables_ptr = block_tables.data_ptr<int>();
955
int* context_lens_ptr = context_lens.data_ptr<int>();
885
-#if 0
886
- T* qk_out_ptr = reinterpret_cast<T*>(qk_out.data_ptr());
887
- T* softmax_out_ptr = reinterpret_cast<T*>(softmax_out.data_ptr());
888
-#endif
956
957
const int max_ctx_blocks = DIVIDE_ROUND_UP(max_context_len, BLOCK_SIZE);
958
const int max_num_partitions =
@@ -972,32 +1039,32 @@ void paged_attention_custom_launcher(
1039
}
1040
}
1041
975
-#define CALL_CUSTOM_LAUNCHER(T, BLK_SIZE, HEAD_SIZE) \
976
- paged_attention_custom_launcher<T, BLK_SIZE, HEAD_SIZE>( \
977
- out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
978
- num_kv_heads, scale, block_tables, context_lens, max_context_len, \
979
- alibi_slopes);
1042
+#define CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE) \
1043
+ paged_attention_custom_launcher<T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE>( \
1044
+ out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
1045
+ num_kv_heads, scale, block_tables, context_lens, max_context_len, \
1046
+ alibi_slopes, k_scale, v_scale);
1047
981
-#define CALL_CUSTOM_LAUNCHER_BLK(T, HEAD_SIZE) \
1048
+#define CALL_CUSTOM_LAUNCHER_BLK(T, KVT, KV_DTYPE, HEAD_SIZE) \
1049
switch (block_size) { \
1050
case 16: \
984
- CALL_CUSTOM_LAUNCHER(T, 16, HEAD_SIZE); \
1051
+ CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, 16, HEAD_SIZE); \
1052
break; \
1053
case 32: \
987
- CALL_CUSTOM_LAUNCHER(T, 32, HEAD_SIZE); \
1054
+ CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, 32, HEAD_SIZE); \
1055
break; \
1056
default: \
1057
TORCH_CHECK(false, "Unsupported block size: ", block_size); \
1058
break; \
1059
}
1060
994
-#define CALL_CUSTOM_LAUNCHER_BLK_HEAD(T) \
1061
+#define CALL_CUSTOM_LAUNCHER_BLK_HEAD(T, KVT, KV_DTYPE) \
1062
switch (head_size) { \
1063
case 64: \
997
- CALL_CUSTOM_LAUNCHER_BLK(T, 64); \
1064
+ CALL_CUSTOM_LAUNCHER_BLK(T, KVT, KV_DTYPE, 64); \
1065
break; \
1066
case 128: \
1000
- CALL_CUSTOM_LAUNCHER_BLK(T, 128); \
1067
+ CALL_CUSTOM_LAUNCHER_BLK(T, KVT, KV_DTYPE, 128); \
1068
break; \
1069
default: \
1070
TORCH_CHECK(false, "Unsupported head size: ", head_size); \
@@ -1020,19 +1087,34 @@ void paged_attention(
1087
torch::Tensor& context_lens, // [num_seqs]
1088
int64_t block_size, int64_t max_context_len,
1089
const c10::optional<torch::Tensor>& alibi_slopes,
1023
- const std::string& kv_cache_dtype) {
1024
- assert(kv_cache_dtype == "auto");
1090
+ const std::string& kv_cache_dtype, double k_scale, double v_scale) {
1091
const int head_size = query.size(2);
1026
- if (query.dtype() == at::ScalarType::Half) {
1027
- CALL_CUSTOM_LAUNCHER_BLK_HEAD(_Float16);
1028
- } else if (query.dtype() == at::ScalarType::BFloat16) {
1029
- CALL_CUSTOM_LAUNCHER_BLK_HEAD(__hip_bfloat16);
1092
+ if (kv_cache_dtype == "auto") {
1093
+ if (query.dtype() == at::ScalarType::Half) {
1094
+ CALL_CUSTOM_LAUNCHER_BLK_HEAD(_Float16, _Float16,
1095
+ vllm::Fp8KVCacheDataType::kAuto);
1096
+ } else if (query.dtype() == at::ScalarType::BFloat16) {
1097
+ CALL_CUSTOM_LAUNCHER_BLK_HEAD(__hip_bfloat16, __hip_bfloat16,
1098
+ vllm::Fp8KVCacheDataType::kAuto);
1099
+ } else {
1100
+ TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
1101
+ }
1102
+ } else if (kv_cache_dtype == "fp8" || kv_cache_dtype == "fp8_e4m3") {
1103
+ if (query.dtype() == at::ScalarType::Half) {
1104
+ CALL_CUSTOM_LAUNCHER_BLK_HEAD(_Float16, uint8_t,
1105
+ vllm::Fp8KVCacheDataType::kFp8E4M3);
1106
+ } else if (query.dtype() == at::ScalarType::BFloat16) {
1107
+ CALL_CUSTOM_LAUNCHER_BLK_HEAD(__hip_bfloat16, uint8_t,
1108
+ vllm::Fp8KVCacheDataType::kFp8E4M3);
1109
+ } else {
1110
+ TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
1111
+ }
1112
} else {
1031
- TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
1113
+ TORCH_CHECK(false, "Unsupported KV cache dtype: ", kv_cache_dtype);
1114
}
1115
}
1116
1117
#undef WARP_SIZE
1118
#undef MAX
1119
#undef MIN
1038
-#undef DIVIDE_ROUND_UP
1120
+#undef DIVIDE_ROUND_UP
\ No newline at end of file
csrc/rocm/ops.h
+2
-1
@@ -10,4 +10,5 @@ void paged_attention(torch::Tensor& out, torch::Tensor& exp_sums,
10
torch::Tensor& context_lens, int64_t block_size,
11
int64_t max_context_len,
12
const c10::optional<torch::Tensor>& alibi_slopes,
13
- const std::string& kv_cache_dtype);
13
+ const std::string& kv_cache_dtype, double k_scale,
14
+ double v_scale);
csrc/rocm/torch_bindings.cpp
+2
-1
@@ -26,7 +26,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, rocm_ops) {
26
" Tensor context_lens, int block_size,"
27
" int max_context_len,"
28
" Tensor? alibi_slopes,"
29
- " str kv_cache_dtype) -> ()");
29
+ " str kv_cache_dtype,"
30
+ " float k_scale, float v_scale) -> ()");
31
rocm_ops.impl("paged_attention", torch::kCUDA, &paged_attention);
32
}
33
tests/kernels/test_attention.py
+63
-188
@@ -31,8 +31,7 @@ NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
31
32
# FlashAttention forward only supports head dimension at most 128
33
# https://github.com/ROCmSoftwarePlatform/flash-attention/blob/3d2b6f5d037782cc2c906909a46fb7e2e1b48b25/csrc/flash_attn_rocm/flash_api.cpp#L62
34
-HEAD_SIZES = [64, 80, 96, 112, 120, 128, 192, 256
35
- ] if not is_hip() else [64, 80, 96, 112, 128]
34
+HEAD_SIZES = [64, 80, 96, 112, 120, 128, 192, 256]
35
36
BLOCK_SIZES = [16, 32]
37
USE_ALIBI = [False, True]
@@ -114,7 +113,8 @@ def ref_single_query_cached_kv_attention(
113
output[i].copy_(out, non_blocking=True)
114
115
117
-@pytest.mark.parametrize("version", ["v1", "v2"])
116
+@pytest.mark.parametrize(
117
+ "version", ["v1", "v2"] if not is_hip() else ["v1", "v2", "rocm"])
118
@pytest.mark.parametrize("num_seqs", NUM_GEN_SEQS)
119
@pytest.mark.parametrize("num_heads", NUM_HEADS)
120
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@@ -137,7 +137,8 @@ def test_paged_attention(
137
seed: int,
138
device: str,
139
) -> None:
140
- if kv_cache_dtype == "fp8" and head_size % 16:
140
+ if ((kv_cache_dtype == "fp8" and head_size % 16)
141
+ or (version == "rocm" and head_size not in (64, 128))):
142
pytest.skip()
143
144
seed_everything(seed)
@@ -206,7 +207,7 @@ def test_paged_attention(
207
kv_cache_dtype, k_scale, v_scale, 0, 0, 0, 64, 0),
208
cond=(head_size == HEAD_SIZES[0]))
209
209
- elif version == "v2":
210
+ elif version in ("v2", "rocm"):
211
num_partitions = ((max_seq_len + PARTITION_SIZE - 1) // PARTITION_SIZE)
212
assert PARTITION_SIZE % block_size == 0
213
num_seqs, num_heads, head_size = output.shape
@@ -219,32 +220,61 @@ def test_paged_attention(
220
dtype=torch.float32,
221
)
222
max_logits = torch.empty_like(exp_sums)
222
- ops.paged_attention_v2(
223
- output,
224
- exp_sums,
225
- max_logits,
226
- tmp_output,
227
- query,
228
- key_cache,
229
- value_cache,
230
- num_kv_heads,
231
- scale,
232
- block_tables,
233
- seq_lens,
234
- block_size,
235
- max_seq_len,
236
- alibi_slopes,
237
- kv_cache_dtype,
238
- k_scale,
239
- v_scale,
240
- )
241
-
242
- opcheck(torch.ops._C.paged_attention_v2,
243
- (output, exp_sums, max_logits, tmp_output, query, key_cache,
244
- value_cache, num_kv_heads, scale, block_tables, seq_lens,
245
- block_size, max_seq_len, alibi_slopes, kv_cache_dtype,
246
- k_scale, v_scale, 0, 0, 0, 64, 0),
247
- cond=(head_size == HEAD_SIZES[0]))
223
+ if version == "v2":
224
+ ops.paged_attention_v2(
225
+ output,
226
+ exp_sums,
227
+ max_logits,
228
+ tmp_output,
229
+ query,
230
+ key_cache,
231
+ value_cache,
232
+ num_kv_heads,
233
+ scale,
234
+ block_tables,
235
+ seq_lens,
236
+ block_size,
237
+ max_seq_len,
238
+ alibi_slopes,
239
+ kv_cache_dtype,
240
+ k_scale,
241
+ v_scale,
242
+ )
243
+
244
+ opcheck(torch.ops._C.paged_attention_v2,
245
+ (output, exp_sums, max_logits, tmp_output, query,
246
+ key_cache, value_cache, num_kv_heads, scale, block_tables,
247
+ seq_lens, block_size, max_seq_len, alibi_slopes,
248
+ kv_cache_dtype, k_scale, v_scale, 0, 0, 0, 64, 0),
249
+ cond=(head_size == HEAD_SIZES[0]))
250
+
251
+ else:
252
+ ops.paged_attention_rocm(
253
+ output,
254
+ exp_sums,
255
+ max_logits,
256
+ tmp_output,
257
+ query,
258
+ key_cache,
259
+ value_cache,
260
+ num_kv_heads,
261
+ scale,
262
+ block_tables,
263
+ seq_lens,
264
+ block_size,
265
+ max_seq_len,
266
+ alibi_slopes,
267
+ kv_cache_dtype,
268
+ k_scale,
269
+ v_scale,
270
+ )
271
+
272
+ opcheck(torch.ops._rocm_C.paged_attention,
273
+ (output, exp_sums, max_logits, tmp_output, query,
274
+ key_cache, value_cache, num_kv_heads, scale, block_tables,
275
+ seq_lens, block_size, max_seq_len, alibi_slopes,
276
+ kv_cache_dtype, k_scale, v_scale),
277
+ cond=(head_size == HEAD_SIZES[0]))
278
279
else:
280
raise AssertionError(f"Unknown version: {version}")
@@ -328,162 +358,6 @@ def ref_multi_query_kv_attention(
358
return torch.cat(ref_outputs, dim=0)
359
360
331
-@pytest.mark.parametrize("version", ["rocm"])
332
-@pytest.mark.parametrize("num_seqs", NUM_GEN_SEQS)
333
-@pytest.mark.parametrize("num_heads", NUM_HEADS)
334
-@pytest.mark.parametrize("head_size", [64, 128]) # only test 64 128
335
-@pytest.mark.parametrize("use_alibi", USE_ALIBI)
336
-@pytest.mark.parametrize("block_size", BLOCK_SIZES)
337
-@pytest.mark.parametrize("dtype", DTYPES)
338
-@pytest.mark.parametrize("kv_cache_dtype", ["auto"])
339
-@pytest.mark.parametrize("seed", SEEDS)
340
-@pytest.mark.parametrize("device", CUDA_DEVICES)
341
-@pytest.mark.skipif(not is_hip(), reason="only for rocm")
342
-def test_paged_attention_rocm(
343
- kv_cache_factory,
344
- version: str,
345
- num_seqs: int,
346
- num_heads: Tuple[int, int],
347
- head_size: int,
348
- use_alibi: bool,
349
- block_size: int,
350
- dtype: torch.dtype,
351
- kv_cache_dtype: str,
352
- seed: int,
353
- device: str,
354
-) -> None:
355
- seed_everything(seed)
356
- torch.set_default_device(device)
357
- scale = float(1.0 / (head_size**0.5))
358
- num_query_heads, num_kv_heads = num_heads
359
- query = torch.empty(num_seqs, num_query_heads, head_size, dtype=dtype)
360
- query.uniform_(-scale, scale)
361
-
362
- assert num_query_heads % num_kv_heads == 0
363
- num_queries_per_kv = num_query_heads // num_kv_heads
364
- alibi_slopes = None
365
- if use_alibi:
366
- alibi_slopes = torch.randn(num_query_heads, dtype=torch.float)
367
-
368
- context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
369
- context_lens[-1] = MAX_SEQ_LEN
370
- #context_lens = [8192 for _ in range(num_seqs)]
371
- max_context_len = max(context_lens)
372
- context_lens = torch.tensor(context_lens, dtype=torch.int)
373
- #print('>>> ctx lens', context_lens)
374
-
375
- # Create the block tables.
376
- max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
377
- block_tables = []
378
- for _ in range(num_seqs):
379
- block_table = [
380
- random.randint(0, NUM_BLOCKS - 1)
381
- for _ in range(max_num_blocks_per_seq)
382
- ]
383
- block_tables.append(block_table)
384
- block_tables = torch.tensor(block_tables, dtype=torch.int)
385
-
386
- # Create the KV caches.
387
- key_caches, value_caches = kv_cache_factory(NUM_BLOCKS, block_size, 1,
388
- num_kv_heads, head_size,
389
- kv_cache_dtype, dtype, seed,
390
- device)
391
- key_cache, value_cache = key_caches[0], value_caches[0]
392
-
393
- # TODO(charlifu) enable fp8 kv cache
394
- # Using default kv_scale
395
- # kv_scale = 1.0
396
-
397
- # Call the paged attention kernel.
398
- output = torch.empty_like(query)
399
- PARTITION_SIZE_ROCM = 256
400
- num_partitions = ((max_context_len + PARTITION_SIZE_ROCM - 1) //
401
- PARTITION_SIZE_ROCM)
402
- assert PARTITION_SIZE_ROCM % block_size == 0
403
- num_seqs, num_heads, head_size = output.shape
404
- tmp_output = torch.empty(
405
- size=(num_seqs, num_heads, num_partitions, head_size),
406
- dtype=output.dtype,
407
- )
408
- exp_sums = torch.empty(
409
- size=(num_seqs, num_heads, num_partitions),
410
- dtype=torch.float32,
411
- )
412
- max_logits = torch.empty_like(exp_sums)
413
- if version == "rocm":
414
- ops.paged_attention_rocm(
415
- output,
416
- exp_sums,
417
- max_logits,
418
- tmp_output,
419
- query,
420
- key_cache,
421
- value_cache,
422
- num_kv_heads,
423
- scale,
424
- block_tables,
425
- context_lens,
426
- block_size,
427
- max_context_len,
428
- alibi_slopes,
429
- kv_cache_dtype,
430
- )
431
- else:
432
- raise AssertionError(f"Unknown version: {version}")
433
-
434
- # Run the reference implementation.
435
- if kv_cache_dtype == "fp8":
436
- # Convert cache data back to dtype.
437
- x = 16 // torch.tensor([], dtype=dtype).element_size()
438
- key_cache_shape = (NUM_BLOCKS, num_kv_heads, head_size // x,
439
- block_size, x)
440
- dequantized_key_cache = torch.empty(size=key_cache_shape,
441
- dtype=dtype,
442
- device=device)
443
- ops.convert_fp8(key_cache, dequantized_key_cache)
444
- key_cache = dequantized_key_cache
445
-
446
- value_cache_shape = value_cache.shape
447
- dequantized_value_cache = torch.empty(size=value_cache_shape,
448
- dtype=dtype,
449
- device=device)
450
- ops.convert_fp8(value_cache, dequantized_value_cache)
451
- value_cache = dequantized_value_cache
452
-
453
- ref_output = torch.empty_like(query)
454
- ref_single_query_cached_kv_attention(
455
- ref_output,
456
- query,
457
- num_queries_per_kv,
458
- key_cache,
459
- value_cache,
460
- block_tables,
461
- context_lens,
462
- scale,
463
- alibi_slopes,
464
- )
465
-
466
- # NOTE(woosuk): Due to the kernel-level differences in the two
467
- # implementations, there is a small numerical difference in the two
468
- # outputs. Thus, we use a relaxed tolerance for the test.
469
- atol = get_default_atol(output) if is_hip() else 1e-3
470
- rtol = get_default_rtol(output) if is_hip() else 1e-5
471
-
472
- # NOTE(zhaoyang): FP8 KV Cache will introduce quantization error,
473
- # so we use a relaxed tolerance for the test.
474
- atol, rtol = 1e-4, 1e-5
475
- if dtype == torch.bfloat16:
476
- atol, rtol = 2e-4, 1e-5
477
- if use_alibi:
478
- if dtype == torch.half:
479
- atol, rtol = 5e-4, 1e-5
480
- if dtype == torch.bfloat16:
481
- atol, rtol = 1e-3, 1e-5
482
- if kv_cache_dtype == "fp8":
483
- atol, rtol = 1e-2, 1e-5
484
- assert torch.allclose(output, ref_output, atol=atol, rtol=rtol)
485
-
486
-
361
# TODO(woosuk): Add tests for USE_ALIBI=True.
362
@pytest.mark.parametrize("num_seqs", NUM_PREFILL_SEQS)
363
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@@ -491,7 +365,8 @@ def test_paged_attention_rocm(
365
@pytest.mark.parametrize("dtype", DTYPES)
366
@pytest.mark.parametrize("seed", SEEDS)
367
@pytest.mark.parametrize("device", CUDA_DEVICES)
494
-@pytest.mark.skipif(is_hip(), reason="skip for rocm")
368
+@pytest.mark.skipif(is_hip(),
369
+ reason="Xformers backend is not supported on ROCm.")
370
@torch.inference_mode()
371
def test_multi_query_kv_attention(
372
num_seqs: int,
vllm/_custom_ops.py
+3
-1
@@ -146,12 +146,14 @@ def paged_attention_rocm(
146
max_seq_len: int,
147
alibi_slopes: Optional[torch.Tensor],
148
kv_cache_dtype: str,
149
+ k_scale: float,
150
+ v_scale: float,
151
) -> None:
152
torch.ops._rocm_C.paged_attention(out, exp_sum, max_logits, tmp_out, query,
153
key_cache, value_cache, num_kv_heads,
154
scale, block_tables, seq_lens,
155
block_size, max_seq_len, alibi_slopes,
154
- kv_cache_dtype)
156
+ kv_cache_dtype, k_scale, v_scale)
157
158
159
# pos encoding ops
vllm/attention/backends/rocm_flash_attn.py
+15
-13
@@ -17,8 +17,8 @@ from vllm.platforms import current_platform
17
18
logger = init_logger(__name__)
19
20
-_PARTITION_SIZE = 256
21
-ON_NAVI = "gfx1" in torch.cuda.get_device_properties("cuda").gcnArchName
20
+_PARTITION_SIZE_ROCM = 512
21
+_ON_NAVI = "gfx1" in torch.cuda.get_device_properties("cuda").gcnArchName
22
23
24
class ROCmFlashAttentionBackend(AttentionBackend):
@@ -489,14 +489,15 @@ class ROCmFlashAttentionImpl(AttentionImpl):
489
num_seqs, num_heads, head_size = decode_query.shape
490
block_size = value_cache.shape[3]
491
gqa_ratio = num_heads // self.num_kv_heads
492
- use_custom = use_rocm_custom_paged_attention(
493
- decode_query.dtype, head_size, block_size, self.kv_cache_dtype,
494
- gqa_ratio, decode_meta.max_decode_seq_len)
492
+ use_custom = _use_rocm_custom_paged_attention(
493
+ decode_query.dtype, head_size, block_size, gqa_ratio,
494
+ decode_meta.max_decode_seq_len)
495
if use_custom:
496
max_seq_len = decode_meta.max_decode_seq_len
497
- max_num_partitions = ((max_seq_len + _PARTITION_SIZE - 1) //
498
- _PARTITION_SIZE)
499
- assert _PARTITION_SIZE % block_size == 0
497
+ max_num_partitions = (
498
+ (max_seq_len + _PARTITION_SIZE_ROCM - 1) //
499
+ _PARTITION_SIZE_ROCM)
500
+ assert _PARTITION_SIZE_ROCM % block_size == 0
501
tmp_output = torch.empty(
502
size=(num_seqs, num_heads, max_num_partitions, head_size),
503
dtype=output.dtype,
@@ -524,6 +525,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
525
max_seq_len,
526
self.alibi_slopes,
527
self.kv_cache_dtype,
528
+ k_scale,
529
+ v_scale,
530
)
531
else:
532
output[num_prefill_tokens:] = PagedAttention.forward_decode(
@@ -580,12 +583,11 @@ def _sdpa_attention(
583
return output
584
585
583
-def use_rocm_custom_paged_attention(qtype: torch.dtype, head_size: int,
584
- block_size: int, kv_cache_dtype: str,
585
- gqa_ratio: int, max_seq_len: int) -> bool:
586
+def _use_rocm_custom_paged_attention(qtype: torch.dtype, head_size: int,
587
+ block_size: int, gqa_ratio: int,
588
+ max_seq_len: int) -> bool:
589
# rocm custom page attention not support on navi (gfx1*)
587
- return (not ON_NAVI and (qtype == torch.half or qtype == torch.bfloat16)
590
+ return (not _ON_NAVI and (qtype == torch.half or qtype == torch.bfloat16)
591
and (head_size == 64 or head_size == 128)
592
and (block_size == 16 or block_size == 32)
590
- and kv_cache_dtype == "auto"
593
and (gqa_ratio >= 1 and gqa_ratio <= 16) and max_seq_len <= 32768)