@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9cc373f390
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Kernel][Amd] Add fp8 kv cache support for rocm custom paged attention (#8577)

Charlie Fu committed Sep 19, 2024 at 12:37 UTC 9cc373f39036af789fb1ffc1e06b23766996d3f4
6 files changed +246 -283
csrc/rocm/attention.cu
+161 -79
@@ -18,8 +18,11 @@
18 #include <ATen/cuda/CUDAContext.h>
19 #include <c10/cuda/CUDAGuard.h>
20 #include <hip/hip_bf16.h>
21 +#include "cuda_compat.h"
22
23 #include <algorithm>
24 +#include "../attention/dtype_fp8.cuh"
25 +#include "../quantization/fp8/amd/quant_utils.cuh"
26
27 #if defined(__HIPCC__) && (defined(__gfx90a__) || defined(__gfx940__) || \
28 defined(__gfx941__) || defined(__gfx942__))
@@ -38,7 +41,6 @@
41 #define MAX(a, b) ((a) > (b) ? (a) : (b))
42 #define MIN(a, b) ((a) < (b) ? (a) : (b))
43 #define DIVIDE_ROUND_UP(a, b) (((a) + (b) - 1) / (b))
41 -#define WARP_SIZE 64
44
45 #if defined(__HIP__MI300_MI250__) // TODO: Add NAVI support
46
@@ -60,6 +62,8 @@ typedef struct _B16x8 {
62 _B16x4 xy[2];
63 } _B16x8;
64
65 +using _B8x8 = uint2;
66 +
67 ////// Non temporal load stores ///////
68
69 template <typename T>
@@ -168,18 +172,40 @@ __device__ __forceinline__ _B16x4 addx4(const _B16x4& inp1,
172 }
173 }
174
175 +template <typename T, vllm::Fp8KVCacheDataType KV_DTYPE>
176 +__device__ __forceinline__ _B16x8 scaled_convert_b8x8(const _B8x8 input,
177 + const float scale) {
178 + union alignas(16) {
179 + uint4 u4;
180 + _B16x8 u16x8;
181 + vllm::bf16_8_t b16x8;
182 + } tmp;
183 + if constexpr (std::is_same<T, _Float16>::value) {
184 + tmp.u4 = vllm::fp8::scaled_convert<uint4, _B8x8, KV_DTYPE>(input, scale);
185 + return tmp.u16x8;
186 + } else if constexpr (std::is_same<T, __hip_bfloat16>::value) {
187 + tmp.b16x8 = vllm::fp8::scaled_convert<vllm::bf16_8_t, _B8x8, KV_DTYPE>(
188 + input, scale);
189 + return tmp.u16x8;
190 + } else {
191 + static_assert(false, "unsupported 16b dtype");
192 + }
193 +}
194 +
195 ///////////////////////////////////////
196
197 // grid (num_seqs, num_partitions,num_heads/gqa_ratio)
198 // block (partition size)
175 -template <typename scalar_t, int BLOCK_SIZE, int HEAD_SIZE, int NUM_THREADS,
199 +template <typename scalar_t, typename cache_t,
200 + vllm::Fp8KVCacheDataType KV_DTYPE, int BLOCK_SIZE, int HEAD_SIZE,
201 + int NUM_THREADS,
202 int GQA_RATIO>
203 __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
178 - const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
179 - const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
180 - // head_size/x, block_size, x]
181 - const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
182 - // head_size, block_size]
204 + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
205 + const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
206 + // head_size/x, block_size, x]
207 + const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
208 + // head_size, block_size]
209 const int num_kv_heads, const float scale,
210 const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
211 const int* __restrict__ context_lens, // [num_seqs]
@@ -192,10 +218,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
218 scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions,
219 // head_size]
220 scalar_t* __restrict__ final_out, // [num_seqs, num_heads, head_size]
195 - #if 0
196 - scalar_t* __restrict__ qk_out, // [num_heads, num_seqs, max_ctx_blocks,block_size]
197 - #endif
198 - int max_ctx_blocks) {
221 + int max_ctx_blocks, float k_scale, float v_scale) {
222 constexpr int NWARPS = NUM_THREADS / WARP_SIZE;
223 const int warpid = threadIdx.x / WARP_SIZE;
224 const int laneid = threadIdx.x % WARP_SIZE;
@@ -222,12 +245,14 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
245 constexpr int x = 16 / sizeof(scalar_t);
246 constexpr int KHELOOP = HEAD_SIZE / x;
247 _B16x8 Klocal[KHELOOP];
248 + _B8x8 Klocalb8[KHELOOP];
249 constexpr int VHELOOP =
250 HEAD_SIZE /
251 WARP_SIZE; // v head_size dimension is distributed across lanes
252 constexpr int VTLOOP = 8; // 16 separate 4xtokens across warp -> 16/2
253 // 8xtokens
254 _B16x8 Vlocal[VHELOOP][VTLOOP];
255 + _B8x8 Vlocalb8[VHELOOP][VTLOOP];
256 floatx4 dout[QHLOOP];
257 float qk_max[QHLOOP];
258 #pragma unroll
@@ -279,6 +304,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
304 (vblock_idx <= last_ctx_block) ? vblock_idx : last_ctx_block;
305 vphysical_blocks[b] = block_table[vblock_idx_ctx];
306 }
307 +
308 // each 4 lanes fetch 8 helems, so warp fetches 8*16 = 128 helems
309 const scalar_t* q_ptr =
310 q + seq_idx * q_stride + wg_start_head_idx * HEAD_SIZE;
@@ -298,17 +324,29 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
324 Qlocal[QHLOOP - 1].xy[1] = {0};
325 }
326
301 - const scalar_t* k_ptr = k_cache + physical_block_number * kv_block_stride +
302 - wg_start_kv_head_idx * kv_head_stride;
327 + const cache_t* k_ptr = k_cache + physical_block_number * kv_block_stride +
328 + wg_start_kv_head_idx * kv_head_stride;
329
330 const int physical_block_offset =
331 local_token_idx % BLOCK_SIZE; // since x=half8, physical_block_offset
332 // is already cast as _H8
307 -
308 - const _B16x8* k_ptrh8 = reinterpret_cast<const _B16x8*>(k_ptr);
333 + if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) {
334 + const _B16x8* k_ptrh8 = reinterpret_cast<const _B16x8*>(k_ptr);
335 + #pragma unroll
336 + for (int d = 0; d < KHELOOP; d++) {
337 + Klocal[d] = k_ptrh8[d * BLOCK_SIZE + physical_block_offset];
338 + }
339 + } else {
340 + constexpr int X = 16 / sizeof(cache_t);
341 + const cache_t* k_ptr2 = k_ptr + physical_block_offset * X;
342 #pragma unroll
310 - for (int d = 0; d < KHELOOP; d++) {
311 - Klocal[d] = k_ptrh8[d * BLOCK_SIZE + physical_block_offset];
343 + for (int d = 0; d < KHELOOP; d++) {
344 + const int head_elem = d * 8;
345 + const int offset1 = head_elem / X;
346 + const int offset2 = head_elem % X;
347 + const cache_t* k_ptr3 = k_ptr2 + offset1 * BLOCK_SIZE * X + offset2;
348 + Klocalb8[d] = *reinterpret_cast<const _B8x8*>(k_ptr3);
349 + }
350 }
351
352 float alibi_slope[QHLOOP];
@@ -322,30 +360,66 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
360 }
361 }
362
325 - const scalar_t* v_ptr = v_cache + wg_start_kv_head_idx * kv_head_stride;
326 - const _B16x8* v_ptrh8 = reinterpret_cast<const _B16x8*>(v_ptr);
327 - // iterate over each v block
363 + const cache_t* v_ptr = v_cache + wg_start_kv_head_idx * kv_head_stride;
364 + if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) {
365 + const _B16x8* v_ptrh8 = reinterpret_cast<const _B16x8*>(v_ptr);
366 + // iterate over each v block
367 #pragma unroll
329 - for (int b = 0; b < VBLOCKS; b++) {
330 - // int32 physical_block_number leads to overflow when multiplied with
331 - // kv_block_stride
332 - const int64_t vphysical_block_number =
333 - static_cast<int64_t>(vphysical_blocks[b]);
334 - const _B16x8* v_ptrh8b =
335 - v_ptrh8 + (vphysical_block_number * kv_block_stride) / 8;
336 - // iterate over each head elem (within head_size)
368 + for (int b = 0; b < VBLOCKS; b++) {
369 + // int32 physical_block_number leads to overflow when multiplied with
370 + // kv_block_stride
371 + const int64_t vphysical_block_number =
372 + static_cast<int64_t>(vphysical_blocks[b]);
373 + const _B16x8* v_ptrh8b =
374 + v_ptrh8 + (vphysical_block_number * kv_block_stride) / 8;
375 + // iterate over each head elem (within head_size)
376 + #pragma unroll
377 + for (int h = 0; h < VHELOOP; h++) {
378 + const int head_size_elem = h * WARP_SIZE + laneid;
379 + const _B16x8* v_ptrh8be = v_ptrh8b + head_size_elem * BLOCK_SIZE / 8;
380 + // iterate over all velems within block
381 + #pragma unroll
382 + for (int d = 0; d < BLOCK_SIZE / 8; d++) {
383 + Vlocal[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
384 + }
385 + }
386 + }
387 + } else {
388 + const _B8x8* v_ptrh8 = reinterpret_cast<const _B8x8*>(v_ptr);
389 + // iterate over each v block
390 + #pragma unroll
391 + for (int b = 0; b < VBLOCKS; b++) {
392 + // int32 physical_block_number leads to overflow when multiplied with
393 + // kv_block_stride
394 + const int64_t vphysical_block_number =
395 + static_cast<int64_t>(vphysical_blocks[b]);
396 + const _B8x8* v_ptrh8b =
397 + v_ptrh8 + (vphysical_block_number * kv_block_stride) / 8;
398 + // iterate over each head elem (within head_size)
399 #pragma unroll
338 - for (int h = 0; h < VHELOOP; h++) {
339 - const int head_size_elem = h * WARP_SIZE + laneid;
340 - const _B16x8* v_ptrh8be = v_ptrh8b + head_size_elem * BLOCK_SIZE / 8;
341 - // iterate over all velems within block
400 + for (int h = 0; h < VHELOOP; h++) {
401 + const int head_size_elem = h * WARP_SIZE + laneid;
402 + const _B8x8* v_ptrh8be = v_ptrh8b + head_size_elem * BLOCK_SIZE / 8;
403 + // iterate over all velems within block
404 #pragma unroll
343 - for (int d = 0; d < BLOCK_SIZE / 8; d++) {
344 - Vlocal[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
405 + for (int d = 0; d < BLOCK_SIZE / 8; d++) {
406 + // Vlocalb8[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
407 + const _B8x8 Vlocalb8 = v_ptrh8be[d];
408 + Vlocal[h][b * BLOCK_SIZE / 8 + d] =
409 + scaled_convert_b8x8<scalar_t, KV_DTYPE>(Vlocalb8, v_scale);
410 + }
411 }
412 }
413 }
414
415 + if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) {
416 + #pragma unroll
417 + for (int d = 0; d < KHELOOP; d++) {
418 + Klocal[d] =
419 + scaled_convert_b8x8<scalar_t, KV_DTYPE>(Klocalb8[d], k_scale);
420 + }
421 + }
422 +
423 #pragma unroll
424 for (int h = 0; h < QHLOOP; h++) {
425 dout[h] = gcn_mfma_instr<scalar_t, 4, 0, 0>(Qlocal[h].xy[0],
@@ -794,14 +868,16 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
868
869 #else // !defined(__HIP__MI300_MI250__) TODO: Add NAVI support
870
797 -template <typename scalar_t, int BLOCK_SIZE, int HEAD_SIZE, int NUM_THREADS,
871 +template <typename scalar_t, typename cache_t,
872 + vllm::Fp8KVCacheDataType KV_DTYPE, int BLOCK_SIZE, int HEAD_SIZE,
873 + int NUM_THREADS,
874 int GQA_RATIO>
875 __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
800 - const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
801 - const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
802 - // head_size/x, block_size, x]
803 - const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
804 - // head_size, block_size]
876 + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
877 + const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads,
878 + // head_size/x, block_size, x]
879 + const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads,
880 + // head_size, block_size]
881 const int num_kv_heads, const float scale,
882 const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
883 const int* __restrict__ context_lens, // [num_seqs]
@@ -814,10 +890,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
890 scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions,
891 // head_size]
892 scalar_t* __restrict__ final_out, // [num_seqs, num_heads, head_size]
817 - #if 0
818 - scalar_t* __restrict__ qk_out, // [num_heads, num_seqs, max_ctx_blocks,block_size]
819 - #endif
820 - int max_ctx_blocks) {
893 + int max_ctx_blocks, float k_scale, float v_scale) {
894 UNREACHABLE_CODE
895 }
896
@@ -839,26 +912,24 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
912 #endif // defined(__HIP__MI300_MI250__) TODO: Add NAVI support
913
914 #define LAUNCH_CUSTOM_ATTENTION(GQA_RATIO) \
842 - paged_attention_ll4mi_QKV_kernel<T, BLOCK_SIZE, HEAD_SIZE, NTHR, GQA_RATIO> \
915 + paged_attention_ll4mi_QKV_kernel<T, KVT, KV_DTYPE, BLOCK_SIZE, HEAD_SIZE, \
916 + NTHR, GQA_RATIO> \
917 <<<grid, block, 0, stream>>>( \
918 query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, scale, \
919 block_tables_ptr, context_lens_ptr, max_num_blocks_per_seq, \
920 alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
847 - exp_sums_ptr, max_logits_ptr, tmp_out_ptr, out_ptr, max_ctx_blocks);
921 + exp_sums_ptr, max_logits_ptr, tmp_out_ptr, out_ptr, max_ctx_blocks, \
922 + k_scale, v_scale);
923
849 -template <typename T, int BLOCK_SIZE, int HEAD_SIZE, int PARTITION_SIZE = 256>
924 +template <typename T, typename KVT, vllm::Fp8KVCacheDataType KV_DTYPE,
925 + int BLOCK_SIZE, int HEAD_SIZE, int PARTITION_SIZE = 512>
926 void paged_attention_custom_launcher(
927 torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
928 torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
929 torch::Tensor& value_cache, const int num_kv_heads, float scale,
930 torch::Tensor& block_tables, torch::Tensor& context_lens,
855 - int max_context_len,
856 -#if 0
857 - torch::Tensor& qk_out,
858 - torch::Tensor& softmax_out,
859 -#endif
860 - const c10::optional<torch::Tensor>& alibi_slopes) {
861 -
931 + int max_context_len, const c10::optional<torch::Tensor>& alibi_slopes,
932 + float k_scale, float v_scale) {
933 int num_seqs = query.size(0);
934 int num_heads = query.size(1);
935 int head_size = query.size(2);
@@ -878,14 +949,10 @@ void paged_attention_custom_launcher(
949 float* max_logits_ptr = reinterpret_cast<float*>(max_logits.data_ptr());
950 T* tmp_out_ptr = reinterpret_cast<T*>(tmp_out.data_ptr());
951 T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
881 - T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
882 - T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
952 + KVT* key_cache_ptr = reinterpret_cast<KVT*>(key_cache.data_ptr());
953 + KVT* value_cache_ptr = reinterpret_cast<KVT*>(value_cache.data_ptr());
954 int* block_tables_ptr = block_tables.data_ptr<int>();
955 int* context_lens_ptr = context_lens.data_ptr<int>();
885 -#if 0
886 - T* qk_out_ptr = reinterpret_cast<T*>(qk_out.data_ptr());
887 - T* softmax_out_ptr = reinterpret_cast<T*>(softmax_out.data_ptr());
888 -#endif
956
957 const int max_ctx_blocks = DIVIDE_ROUND_UP(max_context_len, BLOCK_SIZE);
958 const int max_num_partitions =
@@ -972,32 +1039,32 @@ void paged_attention_custom_launcher(
1039 }
1040 }
1041
975 -#define CALL_CUSTOM_LAUNCHER(T, BLK_SIZE, HEAD_SIZE) \
976 - paged_attention_custom_launcher<T, BLK_SIZE, HEAD_SIZE>( \
977 - out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
978 - num_kv_heads, scale, block_tables, context_lens, max_context_len, \
979 - alibi_slopes);
1042 +#define CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE) \
1043 + paged_attention_custom_launcher<T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE>( \
1044 + out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
1045 + num_kv_heads, scale, block_tables, context_lens, max_context_len, \
1046 + alibi_slopes, k_scale, v_scale);
1047
981 -#define CALL_CUSTOM_LAUNCHER_BLK(T, HEAD_SIZE) \
1048 +#define CALL_CUSTOM_LAUNCHER_BLK(T, KVT, KV_DTYPE, HEAD_SIZE) \
1049 switch (block_size) { \
1050 case 16: \
984 - CALL_CUSTOM_LAUNCHER(T, 16, HEAD_SIZE); \
1051 + CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, 16, HEAD_SIZE); \
1052 break; \
1053 case 32: \
987 - CALL_CUSTOM_LAUNCHER(T, 32, HEAD_SIZE); \
1054 + CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, 32, HEAD_SIZE); \
1055 break; \
1056 default: \
1057 TORCH_CHECK(false, "Unsupported block size: ", block_size); \
1058 break; \
1059 }
1060
994 -#define CALL_CUSTOM_LAUNCHER_BLK_HEAD(T) \
1061 +#define CALL_CUSTOM_LAUNCHER_BLK_HEAD(T, KVT, KV_DTYPE) \
1062 switch (head_size) { \
1063 case 64: \
997 - CALL_CUSTOM_LAUNCHER_BLK(T, 64); \
1064 + CALL_CUSTOM_LAUNCHER_BLK(T, KVT, KV_DTYPE, 64); \
1065 break; \
1066 case 128: \
1000 - CALL_CUSTOM_LAUNCHER_BLK(T, 128); \
1067 + CALL_CUSTOM_LAUNCHER_BLK(T, KVT, KV_DTYPE, 128); \
1068 break; \
1069 default: \
1070 TORCH_CHECK(false, "Unsupported head size: ", head_size); \
@@ -1020,19 +1087,34 @@ void paged_attention(
1087 torch::Tensor& context_lens, // [num_seqs]
1088 int64_t block_size, int64_t max_context_len,
1089 const c10::optional<torch::Tensor>& alibi_slopes,
1023 - const std::string& kv_cache_dtype) {
1024 - assert(kv_cache_dtype == "auto");
1090 + const std::string& kv_cache_dtype, double k_scale, double v_scale) {
1091 const int head_size = query.size(2);
1026 - if (query.dtype() == at::ScalarType::Half) {
1027 - CALL_CUSTOM_LAUNCHER_BLK_HEAD(_Float16);
1028 - } else if (query.dtype() == at::ScalarType::BFloat16) {
1029 - CALL_CUSTOM_LAUNCHER_BLK_HEAD(__hip_bfloat16);
1092 + if (kv_cache_dtype == "auto") {
1093 + if (query.dtype() == at::ScalarType::Half) {
1094 + CALL_CUSTOM_LAUNCHER_BLK_HEAD(_Float16, _Float16,
1095 + vllm::Fp8KVCacheDataType::kAuto);
1096 + } else if (query.dtype() == at::ScalarType::BFloat16) {
1097 + CALL_CUSTOM_LAUNCHER_BLK_HEAD(__hip_bfloat16, __hip_bfloat16,
1098 + vllm::Fp8KVCacheDataType::kAuto);
1099 + } else {
1100 + TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
1101 + }
1102 + } else if (kv_cache_dtype == "fp8" || kv_cache_dtype == "fp8_e4m3") {
1103 + if (query.dtype() == at::ScalarType::Half) {
1104 + CALL_CUSTOM_LAUNCHER_BLK_HEAD(_Float16, uint8_t,
1105 + vllm::Fp8KVCacheDataType::kFp8E4M3);
1106 + } else if (query.dtype() == at::ScalarType::BFloat16) {
1107 + CALL_CUSTOM_LAUNCHER_BLK_HEAD(__hip_bfloat16, uint8_t,
1108 + vllm::Fp8KVCacheDataType::kFp8E4M3);
1109 + } else {
1110 + TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
1111 + }
1112 } else {
1031 - TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
1113 + TORCH_CHECK(false, "Unsupported KV cache dtype: ", kv_cache_dtype);
1114 }
1115 }
1116
1117 #undef WARP_SIZE
1118 #undef MAX
1119 #undef MIN
1038 -#undef DIVIDE_ROUND_UP
1120 +#undef DIVIDE_ROUND_UP
\ No newline at end of file
csrc/rocm/ops.h
+2 -1
@@ -10,4 +10,5 @@ void paged_attention(torch::Tensor& out, torch::Tensor& exp_sums,
10 torch::Tensor& context_lens, int64_t block_size,
11 int64_t max_context_len,
12 const c10::optional<torch::Tensor>& alibi_slopes,
13 - const std::string& kv_cache_dtype);
13 + const std::string& kv_cache_dtype, double k_scale,
14 + double v_scale);
csrc/rocm/torch_bindings.cpp
+2 -1
@@ -26,7 +26,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, rocm_ops) {
26 " Tensor context_lens, int block_size,"
27 " int max_context_len,"
28 " Tensor? alibi_slopes,"
29 - " str kv_cache_dtype) -> ()");
29 + " str kv_cache_dtype,"
30 + " float k_scale, float v_scale) -> ()");
31 rocm_ops.impl("paged_attention", torch::kCUDA, &paged_attention);
32 }
33
tests/kernels/test_attention.py
+63 -188
@@ -31,8 +31,7 @@ NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
31
32 # FlashAttention forward only supports head dimension at most 128
33 # https://github.com/ROCmSoftwarePlatform/flash-attention/blob/3d2b6f5d037782cc2c906909a46fb7e2e1b48b25/csrc/flash_attn_rocm/flash_api.cpp#L62
34 -HEAD_SIZES = [64, 80, 96, 112, 120, 128, 192, 256
35 - ] if not is_hip() else [64, 80, 96, 112, 128]
34 +HEAD_SIZES = [64, 80, 96, 112, 120, 128, 192, 256]
35
36 BLOCK_SIZES = [16, 32]
37 USE_ALIBI = [False, True]
@@ -114,7 +113,8 @@ def ref_single_query_cached_kv_attention(
113 output[i].copy_(out, non_blocking=True)
114
115
117 -@pytest.mark.parametrize("version", ["v1", "v2"])
116 +@pytest.mark.parametrize(
117 + "version", ["v1", "v2"] if not is_hip() else ["v1", "v2", "rocm"])
118 @pytest.mark.parametrize("num_seqs", NUM_GEN_SEQS)
119 @pytest.mark.parametrize("num_heads", NUM_HEADS)
120 @pytest.mark.parametrize("head_size", HEAD_SIZES)
@@ -137,7 +137,8 @@ def test_paged_attention(
137 seed: int,
138 device: str,
139 ) -> None:
140 - if kv_cache_dtype == "fp8" and head_size % 16:
140 + if ((kv_cache_dtype == "fp8" and head_size % 16)
141 + or (version == "rocm" and head_size not in (64, 128))):
142 pytest.skip()
143
144 seed_everything(seed)
@@ -206,7 +207,7 @@ def test_paged_attention(
207 kv_cache_dtype, k_scale, v_scale, 0, 0, 0, 64, 0),
208 cond=(head_size == HEAD_SIZES[0]))
209
209 - elif version == "v2":
210 + elif version in ("v2", "rocm"):
211 num_partitions = ((max_seq_len + PARTITION_SIZE - 1) // PARTITION_SIZE)
212 assert PARTITION_SIZE % block_size == 0
213 num_seqs, num_heads, head_size = output.shape
@@ -219,32 +220,61 @@ def test_paged_attention(
220 dtype=torch.float32,
221 )
222 max_logits = torch.empty_like(exp_sums)
222 - ops.paged_attention_v2(
223 - output,
224 - exp_sums,
225 - max_logits,
226 - tmp_output,
227 - query,
228 - key_cache,
229 - value_cache,
230 - num_kv_heads,
231 - scale,
232 - block_tables,
233 - seq_lens,
234 - block_size,
235 - max_seq_len,
236 - alibi_slopes,
237 - kv_cache_dtype,
238 - k_scale,
239 - v_scale,
240 - )
241 -
242 - opcheck(torch.ops._C.paged_attention_v2,
243 - (output, exp_sums, max_logits, tmp_output, query, key_cache,
244 - value_cache, num_kv_heads, scale, block_tables, seq_lens,
245 - block_size, max_seq_len, alibi_slopes, kv_cache_dtype,
246 - k_scale, v_scale, 0, 0, 0, 64, 0),
247 - cond=(head_size == HEAD_SIZES[0]))
223 + if version == "v2":
224 + ops.paged_attention_v2(
225 + output,
226 + exp_sums,
227 + max_logits,
228 + tmp_output,
229 + query,
230 + key_cache,
231 + value_cache,
232 + num_kv_heads,
233 + scale,
234 + block_tables,
235 + seq_lens,
236 + block_size,
237 + max_seq_len,
238 + alibi_slopes,
239 + kv_cache_dtype,
240 + k_scale,
241 + v_scale,
242 + )
243 +
244 + opcheck(torch.ops._C.paged_attention_v2,
245 + (output, exp_sums, max_logits, tmp_output, query,
246 + key_cache, value_cache, num_kv_heads, scale, block_tables,
247 + seq_lens, block_size, max_seq_len, alibi_slopes,
248 + kv_cache_dtype, k_scale, v_scale, 0, 0, 0, 64, 0),
249 + cond=(head_size == HEAD_SIZES[0]))
250 +
251 + else:
252 + ops.paged_attention_rocm(
253 + output,
254 + exp_sums,
255 + max_logits,
256 + tmp_output,
257 + query,
258 + key_cache,
259 + value_cache,
260 + num_kv_heads,
261 + scale,
262 + block_tables,
263 + seq_lens,
264 + block_size,
265 + max_seq_len,
266 + alibi_slopes,
267 + kv_cache_dtype,
268 + k_scale,
269 + v_scale,
270 + )
271 +
272 + opcheck(torch.ops._rocm_C.paged_attention,
273 + (output, exp_sums, max_logits, tmp_output, query,
274 + key_cache, value_cache, num_kv_heads, scale, block_tables,
275 + seq_lens, block_size, max_seq_len, alibi_slopes,
276 + kv_cache_dtype, k_scale, v_scale),
277 + cond=(head_size == HEAD_SIZES[0]))
278
279 else:
280 raise AssertionError(f"Unknown version: {version}")
@@ -328,162 +358,6 @@ def ref_multi_query_kv_attention(
358 return torch.cat(ref_outputs, dim=0)
359
360
331 -@pytest.mark.parametrize("version", ["rocm"])
332 -@pytest.mark.parametrize("num_seqs", NUM_GEN_SEQS)
333 -@pytest.mark.parametrize("num_heads", NUM_HEADS)
334 -@pytest.mark.parametrize("head_size", [64, 128]) # only test 64 128
335 -@pytest.mark.parametrize("use_alibi", USE_ALIBI)
336 -@pytest.mark.parametrize("block_size", BLOCK_SIZES)
337 -@pytest.mark.parametrize("dtype", DTYPES)
338 -@pytest.mark.parametrize("kv_cache_dtype", ["auto"])
339 -@pytest.mark.parametrize("seed", SEEDS)
340 -@pytest.mark.parametrize("device", CUDA_DEVICES)
341 -@pytest.mark.skipif(not is_hip(), reason="only for rocm")
342 -def test_paged_attention_rocm(
343 - kv_cache_factory,
344 - version: str,
345 - num_seqs: int,
346 - num_heads: Tuple[int, int],
347 - head_size: int,
348 - use_alibi: bool,
349 - block_size: int,
350 - dtype: torch.dtype,
351 - kv_cache_dtype: str,
352 - seed: int,
353 - device: str,
354 -) -> None:
355 - seed_everything(seed)
356 - torch.set_default_device(device)
357 - scale = float(1.0 / (head_size**0.5))
358 - num_query_heads, num_kv_heads = num_heads
359 - query = torch.empty(num_seqs, num_query_heads, head_size, dtype=dtype)
360 - query.uniform_(-scale, scale)
361 -
362 - assert num_query_heads % num_kv_heads == 0
363 - num_queries_per_kv = num_query_heads // num_kv_heads
364 - alibi_slopes = None
365 - if use_alibi:
366 - alibi_slopes = torch.randn(num_query_heads, dtype=torch.float)
367 -
368 - context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
369 - context_lens[-1] = MAX_SEQ_LEN
370 - #context_lens = [8192 for _ in range(num_seqs)]
371 - max_context_len = max(context_lens)
372 - context_lens = torch.tensor(context_lens, dtype=torch.int)
373 - #print('>>> ctx lens', context_lens)
374 -
375 - # Create the block tables.
376 - max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
377 - block_tables = []
378 - for _ in range(num_seqs):
379 - block_table = [
380 - random.randint(0, NUM_BLOCKS - 1)
381 - for _ in range(max_num_blocks_per_seq)
382 - ]
383 - block_tables.append(block_table)
384 - block_tables = torch.tensor(block_tables, dtype=torch.int)
385 -
386 - # Create the KV caches.
387 - key_caches, value_caches = kv_cache_factory(NUM_BLOCKS, block_size, 1,
388 - num_kv_heads, head_size,
389 - kv_cache_dtype, dtype, seed,
390 - device)
391 - key_cache, value_cache = key_caches[0], value_caches[0]
392 -
393 - # TODO(charlifu) enable fp8 kv cache
394 - # Using default kv_scale
395 - # kv_scale = 1.0
396 -
397 - # Call the paged attention kernel.
398 - output = torch.empty_like(query)
399 - PARTITION_SIZE_ROCM = 256
400 - num_partitions = ((max_context_len + PARTITION_SIZE_ROCM - 1) //
401 - PARTITION_SIZE_ROCM)
402 - assert PARTITION_SIZE_ROCM % block_size == 0
403 - num_seqs, num_heads, head_size = output.shape
404 - tmp_output = torch.empty(
405 - size=(num_seqs, num_heads, num_partitions, head_size),
406 - dtype=output.dtype,
407 - )
408 - exp_sums = torch.empty(
409 - size=(num_seqs, num_heads, num_partitions),
410 - dtype=torch.float32,
411 - )
412 - max_logits = torch.empty_like(exp_sums)
413 - if version == "rocm":
414 - ops.paged_attention_rocm(
415 - output,
416 - exp_sums,
417 - max_logits,
418 - tmp_output,
419 - query,
420 - key_cache,
421 - value_cache,
422 - num_kv_heads,
423 - scale,
424 - block_tables,
425 - context_lens,
426 - block_size,
427 - max_context_len,
428 - alibi_slopes,
429 - kv_cache_dtype,
430 - )
431 - else:
432 - raise AssertionError(f"Unknown version: {version}")
433 -
434 - # Run the reference implementation.
435 - if kv_cache_dtype == "fp8":
436 - # Convert cache data back to dtype.
437 - x = 16 // torch.tensor([], dtype=dtype).element_size()
438 - key_cache_shape = (NUM_BLOCKS, num_kv_heads, head_size // x,
439 - block_size, x)
440 - dequantized_key_cache = torch.empty(size=key_cache_shape,
441 - dtype=dtype,
442 - device=device)
443 - ops.convert_fp8(key_cache, dequantized_key_cache)
444 - key_cache = dequantized_key_cache
445 -
446 - value_cache_shape = value_cache.shape
447 - dequantized_value_cache = torch.empty(size=value_cache_shape,
448 - dtype=dtype,
449 - device=device)
450 - ops.convert_fp8(value_cache, dequantized_value_cache)
451 - value_cache = dequantized_value_cache
452 -
453 - ref_output = torch.empty_like(query)
454 - ref_single_query_cached_kv_attention(
455 - ref_output,
456 - query,
457 - num_queries_per_kv,
458 - key_cache,
459 - value_cache,
460 - block_tables,
461 - context_lens,
462 - scale,
463 - alibi_slopes,
464 - )
465 -
466 - # NOTE(woosuk): Due to the kernel-level differences in the two
467 - # implementations, there is a small numerical difference in the two
468 - # outputs. Thus, we use a relaxed tolerance for the test.
469 - atol = get_default_atol(output) if is_hip() else 1e-3
470 - rtol = get_default_rtol(output) if is_hip() else 1e-5
471 -
472 - # NOTE(zhaoyang): FP8 KV Cache will introduce quantization error,
473 - # so we use a relaxed tolerance for the test.
474 - atol, rtol = 1e-4, 1e-5
475 - if dtype == torch.bfloat16:
476 - atol, rtol = 2e-4, 1e-5
477 - if use_alibi:
478 - if dtype == torch.half:
479 - atol, rtol = 5e-4, 1e-5
480 - if dtype == torch.bfloat16:
481 - atol, rtol = 1e-3, 1e-5
482 - if kv_cache_dtype == "fp8":
483 - atol, rtol = 1e-2, 1e-5
484 - assert torch.allclose(output, ref_output, atol=atol, rtol=rtol)
485 -
486 -
361 # TODO(woosuk): Add tests for USE_ALIBI=True.
362 @pytest.mark.parametrize("num_seqs", NUM_PREFILL_SEQS)
363 @pytest.mark.parametrize("num_heads", NUM_HEADS)
@@ -491,7 +365,8 @@ def test_paged_attention_rocm(
365 @pytest.mark.parametrize("dtype", DTYPES)
366 @pytest.mark.parametrize("seed", SEEDS)
367 @pytest.mark.parametrize("device", CUDA_DEVICES)
494 -@pytest.mark.skipif(is_hip(), reason="skip for rocm")
368 +@pytest.mark.skipif(is_hip(),
369 + reason="Xformers backend is not supported on ROCm.")
370 @torch.inference_mode()
371 def test_multi_query_kv_attention(
372 num_seqs: int,
vllm/_custom_ops.py
+3 -1
@@ -146,12 +146,14 @@ def paged_attention_rocm(
146 max_seq_len: int,
147 alibi_slopes: Optional[torch.Tensor],
148 kv_cache_dtype: str,
149 + k_scale: float,
150 + v_scale: float,
151 ) -> None:
152 torch.ops._rocm_C.paged_attention(out, exp_sum, max_logits, tmp_out, query,
153 key_cache, value_cache, num_kv_heads,
154 scale, block_tables, seq_lens,
155 block_size, max_seq_len, alibi_slopes,
154 - kv_cache_dtype)
156 + kv_cache_dtype, k_scale, v_scale)
157
158
159 # pos encoding ops
vllm/attention/backends/rocm_flash_attn.py
+15 -13
@@ -17,8 +17,8 @@ from vllm.platforms import current_platform
17
18 logger = init_logger(__name__)
19
20 -_PARTITION_SIZE = 256
21 -ON_NAVI = "gfx1" in torch.cuda.get_device_properties("cuda").gcnArchName
20 +_PARTITION_SIZE_ROCM = 512
21 +_ON_NAVI = "gfx1" in torch.cuda.get_device_properties("cuda").gcnArchName
22
23
24 class ROCmFlashAttentionBackend(AttentionBackend):
@@ -489,14 +489,15 @@ class ROCmFlashAttentionImpl(AttentionImpl):
489 num_seqs, num_heads, head_size = decode_query.shape
490 block_size = value_cache.shape[3]
491 gqa_ratio = num_heads // self.num_kv_heads
492 - use_custom = use_rocm_custom_paged_attention(
493 - decode_query.dtype, head_size, block_size, self.kv_cache_dtype,
494 - gqa_ratio, decode_meta.max_decode_seq_len)
492 + use_custom = _use_rocm_custom_paged_attention(
493 + decode_query.dtype, head_size, block_size, gqa_ratio,
494 + decode_meta.max_decode_seq_len)
495 if use_custom:
496 max_seq_len = decode_meta.max_decode_seq_len
497 - max_num_partitions = ((max_seq_len + _PARTITION_SIZE - 1) //
498 - _PARTITION_SIZE)
499 - assert _PARTITION_SIZE % block_size == 0
497 + max_num_partitions = (
498 + (max_seq_len + _PARTITION_SIZE_ROCM - 1) //
499 + _PARTITION_SIZE_ROCM)
500 + assert _PARTITION_SIZE_ROCM % block_size == 0
501 tmp_output = torch.empty(
502 size=(num_seqs, num_heads, max_num_partitions, head_size),
503 dtype=output.dtype,
@@ -524,6 +525,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
525 max_seq_len,
526 self.alibi_slopes,
527 self.kv_cache_dtype,
528 + k_scale,
529 + v_scale,
530 )
531 else:
532 output[num_prefill_tokens:] = PagedAttention.forward_decode(
@@ -580,12 +583,11 @@ def _sdpa_attention(
583 return output
584
585
583 -def use_rocm_custom_paged_attention(qtype: torch.dtype, head_size: int,
584 - block_size: int, kv_cache_dtype: str,
585 - gqa_ratio: int, max_seq_len: int) -> bool:
586 +def _use_rocm_custom_paged_attention(qtype: torch.dtype, head_size: int,
587 + block_size: int, gqa_ratio: int,
588 + max_seq_len: int) -> bool:
589 # rocm custom page attention not support on navi (gfx1*)
587 - return (not ON_NAVI and (qtype == torch.half or qtype == torch.bfloat16)
590 + return (not _ON_NAVI and (qtype == torch.half or qtype == torch.bfloat16)
591 and (head_size == 64 or head_size == 128)
592 and (block_size == 16 or block_size == 32)
590 - and kv_cache_dtype == "auto"
593 and (gqa_ratio >= 1 and gqa_ratio <= 16) and max_seq_len <= 32768)