22
23
#include "attention_dtypes.h"
24
#include "attention_utils.cuh"
25
-#ifdef ENABLE_FP8_E5M2
25
+
26
+#if defined(ENABLE_FP8_E5M2)
27
#include "../quantization/fp8_e5m2_kvcache/quant_utils.cuh"
28
+#elif defined(ENABLE_FP8_E4M3)
29
+#include "../quantization/fp8/amd_detail/quant_utils.cuh"
30
#endif
31
32
#include <algorithm>
33
34
+#ifdef USE_ROCM
35
+ #include <hip/hip_bf16.h>
36
+ typedef __hip_bfloat16 __nv_bfloat16;
37
+#endif
38
+
39
+#ifndef USE_ROCM
40
+#define WARP_SIZE 32
41
+#else
42
+#define WARP_SIZE warpSize
43
+#endif
44
+
45
#define MAX(a, b) ((a) > (b) ? (a) : (b))
46
#define MIN(a, b) ((a) < (b) ? (a) : (b))
47
#define DIVIDE_ROUND_UP(a, b) (((a) + (b) - 1) / (b))
92
int HEAD_SIZE,
93
int BLOCK_SIZE,
94
int NUM_THREADS,
81
- bool IS_FP8_E5M2_KV_CACHE,
95
+ bool IS_FP8_KV_CACHE,
96
int PARTITION_SIZE = 0> // Zero means no partitioning.
97
__device__ void paged_attention_kernel(
98
float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
109
const float* __restrict__ alibi_slopes, // [num_heads]
110
const int q_stride,
111
const int kv_block_stride,
98
- const int kv_head_stride) {
112
+ const int kv_head_stride,
113
+ const float kv_scale) {
114
const int seq_idx = blockIdx.y;
115
const int partition_idx = blockIdx.z;
116
const int max_num_partitions = gridDim.z;
157
constexpr int VEC_SIZE = MAX(16 / (THREAD_GROUP_SIZE * sizeof(scalar_t)), 1);
158
using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
159
using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
145
-#ifdef ENABLE_FP8_E5M2
160
+#if defined(ENABLE_FP8_E5M2) || defined(ENABLE_FP8_E4M3)
161
using Quant_vec = typename Vec<cache_t, VEC_SIZE>::Type;
162
#endif
163
223
const int vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;
224
const int offset1 = (vec_idx * VEC_SIZE) / x;
225
const int offset2 = (vec_idx * VEC_SIZE) % x;
211
- if constexpr (IS_FP8_E5M2_KV_CACHE) {
212
-#ifdef ENABLE_FP8_E5M2
226
+ if constexpr (IS_FP8_KV_CACHE) {
227
+#if defined(ENABLE_FP8_E5M2)
228
Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
229
// Vector conversion from Quant_vec to K_vec.
230
k_vecs[j] = fp8_e5m2_unscaled::vec_conversion<K_vec, Quant_vec>(k_vec_quant);
231
+#elif defined(ENABLE_FP8_E4M3)
232
+ Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
233
+ // Vector conversion from Quant_vec to K_vec. Use scaled_vec_conversion to convert FP8_E4M3 quantized k
234
+ // cache vec to k vec in higher precision (FP16, BFloat16, etc.)
235
+ k_vecs[j] = fp8_e4m3::scaled_vec_conversion<K_vec, Quant_vec>(k_vec_quant, kv_scale);
236
#else
237
assert(false);
238
#endif
312
constexpr int V_VEC_SIZE = MIN(16 / sizeof(scalar_t), BLOCK_SIZE);
313
using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
314
using L_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
295
-#ifdef ENABLE_FP8_E5M2
315
+#if defined(ENABLE_FP8_E5M2) || defined(ENABLE_FP8_E4M3)
316
using V_quant_vec = typename Vec<cache_t, V_VEC_SIZE>::Type;
317
#endif
318
using Float_L_vec = typename FloatVec<L_vec>::Type;
348
if (row_idx < HEAD_SIZE) {
349
const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
350
V_vec v_vec;
331
- if constexpr (IS_FP8_E5M2_KV_CACHE) {
332
-#ifdef ENABLE_FP8_E5M2
351
+ if constexpr (IS_FP8_KV_CACHE) {
352
+#if defined(ENABLE_FP8_E5M2)
353
V_quant_vec v_quant_vec = *reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
354
// Vector conversion from V_quant_vec to V_vec.
355
v_vec = fp8_e5m2_unscaled::vec_conversion<V_vec, V_quant_vec>(v_quant_vec);
356
+#elif defined(ENABLE_FP8_E4M3)
357
+ V_quant_vec v_quant_vec = *reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
358
+ // Vector conversion from V_quant_vec to V_vec. Use scaled_vec_conversion to convert
359
+ // FP8_E4M3 quantized v cache vec to v vec in higher precision (FP16, BFloat16, etc.)
360
+ v_vec = fp8_e4m3::scaled_vec_conversion<V_vec, V_quant_vec>(v_quant_vec, kv_scale);
361
#else
362
assert(false);
363
#endif
448
int HEAD_SIZE,
449
int BLOCK_SIZE,
450
int NUM_THREADS,
426
- bool IS_FP8_E5M2_KV_CACHE>
451
+ bool IS_FP8_KV_CACHE>
452
__global__ void paged_attention_v1_kernel(
453
scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
454
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
462
const float* __restrict__ alibi_slopes, // [num_heads]
463
const int q_stride,
464
const int kv_block_stride,
440
- const int kv_head_stride) {
441
- paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_E5M2_KV_CACHE>(
465
+ const int kv_head_stride,
466
+ const float kv_scale) {
467
+ paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_KV_CACHE>(
468
/* exp_sums */ nullptr, /* max_logits */ nullptr,
469
out, q, k_cache, v_cache, num_kv_heads, scale, block_tables, context_lens,
444
- max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride, kv_head_stride);
470
+ max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride, kv_head_stride, kv_scale);
471
}
472
473
// Grid: (num_heads, num_seqs, max_num_partitions).
477
int HEAD_SIZE,
478
int BLOCK_SIZE,
479
int NUM_THREADS,
454
- bool IS_FP8_E5M2_KV_CACHE,
480
+ bool IS_FP8_KV_CACHE,
481
int PARTITION_SIZE>
482
__global__ void paged_attention_v2_kernel(
483
float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
494
const float* __restrict__ alibi_slopes, // [num_heads]
495
const int q_stride,
496
const int kv_block_stride,
471
- const int kv_head_stride) {
472
- paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_E5M2_KV_CACHE, PARTITION_SIZE>(
497
+ const int kv_head_stride,
498
+ const float kv_scale) {
499
+ paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_KV_CACHE, PARTITION_SIZE>(
500
exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
501
block_tables, context_lens, max_num_blocks_per_seq, alibi_slopes,
475
- q_stride, kv_block_stride, kv_head_stride);
502
+ q_stride, kv_block_stride, kv_head_stride, kv_scale);
503
}
504
505
// Grid: (num_heads, num_seqs).
606
#define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
607
VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
608
((void*)vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
582
- IS_FP8_E5M2_KV_CACHE>), shared_mem_size); \
609
+ IS_FP8_KV_CACHE>), shared_mem_size); \
610
vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
584
- IS_FP8_E5M2_KV_CACHE><<<grid, block, shared_mem_size, stream>>>( \
611
+ IS_FP8_KV_CACHE><<<grid, block, shared_mem_size, stream>>>( \
612
out_ptr, \
613
query_ptr, \
614
key_cache_ptr, \
621
alibi_slopes_ptr, \
622
q_stride, \
623
kv_block_stride, \
597
- kv_head_stride);
624
+ kv_head_stride, \
625
+ kv_scale);
626
627
// TODO(woosuk): Tune NUM_THREADS.
628
template<
629
typename T,
630
typename CACHE_T,
631
int BLOCK_SIZE,
604
- bool IS_FP8_E5M2_KV_CACHE,
632
+ bool IS_FP8_KV_CACHE,
633
int NUM_THREADS = 128>
634
void paged_attention_v1_launcher(
635
torch::Tensor& out,
641
torch::Tensor& block_tables,
642
torch::Tensor& context_lens,
643
int max_context_len,
616
- const c10::optional<torch::Tensor>& alibi_slopes) {
644
+ const c10::optional<torch::Tensor>& alibi_slopes,
645
+ float kv_scale) {
646
int num_seqs = query.size(0);
647
int num_heads = query.size(1);
648
int head_size = query.size(2);
706
}
707
}
708
680
-#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_E5M2_KV_CACHE) \
681
- paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, IS_FP8_E5M2_KV_CACHE>( \
709
+#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
710
+ paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE>( \
711
out, \
712
query, \
713
key_cache, \
717
block_tables, \
718
context_lens, \
719
max_context_len, \
691
- alibi_slopes);
720
+ alibi_slopes, \
721
+ kv_scale);
722
723
// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
724
// 1, 2, 4, 64, 128, 256.
695
-#define CALL_V1_LAUNCHER_BLOCK_SIZE(T, CACHE_T, IS_FP8_E5M2_KV_CACHE) \
725
+#define CALL_V1_LAUNCHER_BLOCK_SIZE(T, CACHE_T, IS_FP8_KV_CACHE) \
726
switch (block_size) { \
727
case 8: \
698
- CALL_V1_LAUNCHER(T, CACHE_T, 8, IS_FP8_E5M2_KV_CACHE); \
728
+ CALL_V1_LAUNCHER(T, CACHE_T, 8, IS_FP8_KV_CACHE); \
729
break; \
730
case 16: \
701
- CALL_V1_LAUNCHER(T, CACHE_T, 16, IS_FP8_E5M2_KV_CACHE); \
731
+ CALL_V1_LAUNCHER(T, CACHE_T, 16, IS_FP8_KV_CACHE); \
732
break; \
733
case 32: \
704
- CALL_V1_LAUNCHER(T, CACHE_T, 32, IS_FP8_E5M2_KV_CACHE); \
734
+ CALL_V1_LAUNCHER(T, CACHE_T, 32, IS_FP8_KV_CACHE); \
735
break; \
736
default: \
737
TORCH_CHECK(false, "Unsupported block size: ", block_size); \
750
int block_size,
751
int max_context_len,
752
const c10::optional<torch::Tensor>& alibi_slopes,
723
- const std::string& kv_cache_dtype) {
753
+ const std::string& kv_cache_dtype,
754
+ float kv_scale) {
755
if (kv_cache_dtype == "auto") {
756
if (query.dtype() == at::ScalarType::Float) {
757
CALL_V1_LAUNCHER_BLOCK_SIZE(float, float, false);
762
} else {
763
TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
764
}
734
- } else if (kv_cache_dtype == "fp8_e5m2") {
765
+ } else if (kv_cache_dtype == "fp8") {
766
if (query.dtype() == at::ScalarType::Float) {
767
CALL_V1_LAUNCHER_BLOCK_SIZE(float, uint8_t, true);
768
} else if (query.dtype() == at::ScalarType::Half) {
779
780
#define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
781
vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
751
- IS_FP8_E5M2_KV_CACHE, PARTITION_SIZE> \
782
+ IS_FP8_KV_CACHE, PARTITION_SIZE> \
783
<<<grid, block, shared_mem_size, stream>>>( \
784
exp_sums_ptr, \
785
max_logits_ptr, \
795
alibi_slopes_ptr, \
796
q_stride, \
797
kv_block_stride, \
767
- kv_head_stride); \
798
+ kv_head_stride, \
799
+ kv_scale); \
800
vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, PARTITION_SIZE> \
801
<<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
802
out_ptr, \
810
typename T,
811
typename CACHE_T,
812
int BLOCK_SIZE,
781
- bool IS_FP8_E5M2_KV_CACHE,
813
+ bool IS_FP8_KV_CACHE,
814
int NUM_THREADS = 128,
815
int PARTITION_SIZE = 512>
816
void paged_attention_v2_launcher(
826
torch::Tensor& block_tables,
827
torch::Tensor& context_lens,
828
int max_context_len,
797
- const c10::optional<torch::Tensor>& alibi_slopes) {
829
+ const c10::optional<torch::Tensor>& alibi_slopes,
830
+ float kv_scale) {
831
int num_seqs = query.size(0);
832
int num_heads = query.size(1);
833
int head_size = query.size(2);
897
}
898
}
899
867
-#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_E5M2_KV_CACHE) \
868
- paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, IS_FP8_E5M2_KV_CACHE>( \
900
+#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
901
+ paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE>( \
902
out, \
903
exp_sums, \
904
max_logits, \
911
block_tables, \
912
context_lens, \
913
max_context_len, \
881
- alibi_slopes);
914
+ alibi_slopes, \
915
+ kv_scale);
916
917
// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
918
// 1, 2, 4, 64, 128, 256.
885
-#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, IS_FP8_E5M2_KV_CACHE) \
919
+#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, IS_FP8_KV_CACHE) \
920
switch (block_size) { \
921
case 8: \
888
- CALL_V2_LAUNCHER(T, CACHE_T, 8, IS_FP8_E5M2_KV_CACHE); \
922
+ CALL_V2_LAUNCHER(T, CACHE_T, 8, IS_FP8_KV_CACHE); \
923
break; \
924
case 16: \
891
- CALL_V2_LAUNCHER(T, CACHE_T, 16, IS_FP8_E5M2_KV_CACHE); \
925
+ CALL_V2_LAUNCHER(T, CACHE_T, 16, IS_FP8_KV_CACHE); \
926
break; \
927
case 32: \
894
- CALL_V2_LAUNCHER(T, CACHE_T, 32, IS_FP8_E5M2_KV_CACHE); \
928
+ CALL_V2_LAUNCHER(T, CACHE_T, 32, IS_FP8_KV_CACHE); \
929
break; \
930
default: \
931
TORCH_CHECK(false, "Unsupported block size: ", block_size); \
947
int block_size,
948
int max_context_len,
949
const c10::optional<torch::Tensor>& alibi_slopes,
916
- const std::string& kv_cache_dtype) {
950
+ const std::string& kv_cache_dtype,
951
+ float kv_scale) {
952
if (kv_cache_dtype == "auto") {
953
if (query.dtype() == at::ScalarType::Float) {
954
CALL_V2_LAUNCHER_BLOCK_SIZE(float, float, false);
959
} else {
960
TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
961
}
927
- } else if (kv_cache_dtype == "fp8_e5m2") {
962
+ } else if (kv_cache_dtype == "fp8") {
963
if (query.dtype() == at::ScalarType::Float) {
964
CALL_V2_LAUNCHER_BLOCK_SIZE(float, uint8_t, true);
965
} else if (query.dtype() == at::ScalarType::Half) {