A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Kernel] Refactor FP8 kv-cache with NVIDIA float8_e4m3 support (#4535)
Cody Yu committed
May 9, 2024 at 17:04 UTC
c83310174055bb124ea2197885b652efd59b7a0f
17 files changed
+843
-558
.buildkite/check-wheel-size.py
+1
-1
@@ -1,7 +1,7 @@
1
import os
2
import zipfile
3
4
-MAX_SIZE_MB = 100
4
+MAX_SIZE_MB = 150
5
6
7
def print_top_10_largest_files(zip_file):
CMakeLists.txt
+1
-1
@@ -167,7 +167,7 @@ set(VLLM_EXT_SRC
167
"csrc/layernorm_kernels.cu"
168
"csrc/quantization/squeezellm/quant_cuda_kernel.cu"
169
"csrc/quantization/gptq/q_gemm.cu"
170
- "csrc/quantization/fp8/fp8_cuda_kernels.cu"
170
+ "csrc/quantization/fp8/common.cu"
171
"csrc/cuda_utils_kernels.cu"
172
"csrc/moe_align_block_size_kernels.cu"
173
"csrc/pybind.cpp")
cmake/utils.cmake
+2
-2
@@ -99,7 +99,7 @@ function (get_torch_gpu_compiler_flags OUT_GPU_FLAGS GPU_LANG)
99
"Failed to determine torch nvcc compiler flags")
100
101
if (CUDA_VERSION VERSION_GREATER_EQUAL 11.8)
102
- list(APPEND GPU_FLAGS "-DENABLE_FP8_E5M2")
102
+ list(APPEND GPU_FLAGS "-DENABLE_FP8")
103
endif()
104
if (CUDA_VERSION VERSION_GREATER_EQUAL 12.0)
105
list(REMOVE_ITEM GPU_FLAGS
@@ -119,7 +119,7 @@ function (get_torch_gpu_compiler_flags OUT_GPU_FLAGS GPU_LANG)
119
120
list(APPEND GPU_FLAGS
121
"-DUSE_ROCM"
122
- "-DENABLE_FP8_E4M3"
122
+ "-DENABLE_FP8"
123
"-U__HIP_NO_HALF_CONVERSIONS__"
124
"-U__HIP_NO_HALF_OPERATORS__"
125
"-fno-gpu-rdc")
csrc/attention/attention_kernels.cu
+110
-176
@@ -19,21 +19,17 @@
19
#include <torch/extension.h>
20
#include <ATen/cuda/CUDAContext.h>
21
#include <c10/cuda/CUDAGuard.h>
22
+#include <algorithm>
23
24
#include "attention_dtypes.h"
25
#include "attention_utils.cuh"
26
26
-#if defined(ENABLE_FP8_E5M2)
27
-#include "../quantization/fp8_e5m2_kvcache/quant_utils.cuh"
28
-#elif defined(ENABLE_FP8_E4M3)
29
-#include "../quantization/fp8/amd_detail/quant_utils.cuh"
30
-#endif
31
-
32
-#include <algorithm>
33
-
27
#ifdef USE_ROCM
28
#include <hip/hip_bf16.h>
29
+ #include "../quantization/fp8/amd/quant_utils.cuh"
30
typedef __hip_bfloat16 __nv_bfloat16;
31
+#else
32
+ #include "../quantization/fp8/nvidia/quant_utils.cuh"
33
#endif
34
35
#ifndef USE_ROCM
@@ -92,7 +88,7 @@ template<
88
int HEAD_SIZE,
89
int BLOCK_SIZE,
90
int NUM_THREADS,
95
- bool IS_FP8_KV_CACHE,
91
+ vllm::Fp8KVCacheDataType KV_DTYPE,
92
int PARTITION_SIZE = 0> // Zero means no partitioning.
93
__device__ void paged_attention_kernel(
94
float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
@@ -157,9 +153,7 @@ __device__ void paged_attention_kernel(
153
constexpr int VEC_SIZE = MAX(16 / (THREAD_GROUP_SIZE * sizeof(scalar_t)), 1);
154
using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
155
using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
160
-#if defined(ENABLE_FP8_E5M2) || defined(ENABLE_FP8_E4M3)
156
using Quant_vec = typename Vec<cache_t, VEC_SIZE>::Type;
162
-#endif
157
158
constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
159
constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
@@ -223,21 +217,14 @@ __device__ void paged_attention_kernel(
217
const int vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;
218
const int offset1 = (vec_idx * VEC_SIZE) / x;
219
const int offset2 = (vec_idx * VEC_SIZE) % x;
226
- if constexpr (IS_FP8_KV_CACHE) {
227
-#if defined(ENABLE_FP8_E5M2)
228
- Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
229
- // Vector conversion from Quant_vec to K_vec.
230
- k_vecs[j] = fp8_e5m2_unscaled::vec_conversion<K_vec, Quant_vec>(k_vec_quant);
231
-#elif defined(ENABLE_FP8_E4M3)
232
- Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
233
- // Vector conversion from Quant_vec to K_vec. Use scaled_vec_conversion to convert FP8_E4M3 quantized k
234
- // cache vec to k vec in higher precision (FP16, BFloat16, etc.)
235
- k_vecs[j] = fp8_e4m3::scaled_vec_conversion<K_vec, Quant_vec>(k_vec_quant, kv_scale);
236
-#else
237
- assert(false);
238
-#endif
239
- } else {
220
+
221
+ if constexpr (KV_DTYPE == Fp8KVCacheDataType::kAuto) {
222
k_vecs[j] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
223
+ } else {
224
+ // Vector conversion from Quant_vec to K_vec.
225
+ Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(
226
+ k_ptr + offset1 * BLOCK_SIZE * x + offset2);
227
+ k_vecs[j] = fp8::scaled_convert<K_vec, Quant_vec, KV_DTYPE>(k_vec_quant, kv_scale);
228
}
229
}
230
@@ -312,9 +299,7 @@ __device__ void paged_attention_kernel(
299
constexpr int V_VEC_SIZE = MIN(16 / sizeof(scalar_t), BLOCK_SIZE);
300
using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
301
using L_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
315
-#if defined(ENABLE_FP8_E5M2) || defined(ENABLE_FP8_E4M3)
302
using V_quant_vec = typename Vec<cache_t, V_VEC_SIZE>::Type;
317
-#endif
303
using Float_L_vec = typename FloatVec<L_vec>::Type;
304
305
constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
@@ -348,21 +333,13 @@ __device__ void paged_attention_kernel(
333
if (row_idx < HEAD_SIZE) {
334
const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
335
V_vec v_vec;
351
- if constexpr (IS_FP8_KV_CACHE) {
352
-#if defined(ENABLE_FP8_E5M2)
336
+
337
+ if constexpr (KV_DTYPE == Fp8KVCacheDataType::kAuto) {
338
+ v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
339
+ } else {
340
V_quant_vec v_quant_vec = *reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
341
// Vector conversion from V_quant_vec to V_vec.
355
- v_vec = fp8_e5m2_unscaled::vec_conversion<V_vec, V_quant_vec>(v_quant_vec);
356
-#elif defined(ENABLE_FP8_E4M3)
357
- V_quant_vec v_quant_vec = *reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
358
- // Vector conversion from V_quant_vec to V_vec. Use scaled_vec_conversion to convert
359
- // FP8_E4M3 quantized v cache vec to v vec in higher precision (FP16, BFloat16, etc.)
360
- v_vec = fp8_e4m3::scaled_vec_conversion<V_vec, V_quant_vec>(v_quant_vec, kv_scale);
361
-#else
362
- assert(false);
363
-#endif
364
- } else {
365
- v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
342
+ v_vec = fp8::scaled_convert<V_vec, V_quant_vec, KV_DTYPE>(v_quant_vec, kv_scale);
343
}
344
if (block_idx == num_seq_blocks - 1) {
345
// NOTE(woosuk): When v_vec contains the tokens that are out of the context,
@@ -448,7 +425,7 @@ template<
425
int HEAD_SIZE,
426
int BLOCK_SIZE,
427
int NUM_THREADS,
451
- bool IS_FP8_KV_CACHE>
428
+ vllm::Fp8KVCacheDataType KV_DTYPE>
429
__global__ void paged_attention_v1_kernel(
430
scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
431
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
@@ -464,7 +441,7 @@ __global__ void paged_attention_v1_kernel(
441
const int kv_block_stride,
442
const int kv_head_stride,
443
const float kv_scale) {
467
- paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_KV_CACHE>(
444
+ paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, KV_DTYPE>(
445
/* exp_sums */ nullptr, /* max_logits */ nullptr,
446
out, q, k_cache, v_cache, num_kv_heads, scale, block_tables, seq_lens,
447
max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride, kv_head_stride, kv_scale);
@@ -477,7 +454,7 @@ template<
454
int HEAD_SIZE,
455
int BLOCK_SIZE,
456
int NUM_THREADS,
480
- bool IS_FP8_KV_CACHE,
457
+ vllm::Fp8KVCacheDataType KV_DTYPE,
458
int PARTITION_SIZE>
459
__global__ void paged_attention_v2_kernel(
460
float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
@@ -496,7 +473,7 @@ __global__ void paged_attention_v2_kernel(
473
const int kv_block_stride,
474
const int kv_head_stride,
475
const float kv_scale) {
499
- paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_KV_CACHE, PARTITION_SIZE>(
476
+ paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, KV_DTYPE, PARTITION_SIZE>(
477
exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
478
block_tables, seq_lens, max_num_blocks_per_seq, alibi_slopes,
479
q_stride, kv_block_stride, kv_head_stride, kv_scale);
@@ -606,9 +583,9 @@ __global__ void paged_attention_v2_reduce_kernel(
583
#define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
584
VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
585
((void*)vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
609
- IS_FP8_KV_CACHE>), shared_mem_size); \
586
+ KV_DTYPE>), shared_mem_size); \
587
vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
611
- IS_FP8_KV_CACHE><<<grid, block, shared_mem_size, stream>>>( \
588
+ KV_DTYPE><<<grid, block, shared_mem_size, stream>>>( \
589
out_ptr, \
590
query_ptr, \
591
key_cache_ptr, \
@@ -629,7 +606,7 @@ template<
606
typename T,
607
typename CACHE_T,
608
int BLOCK_SIZE,
632
- bool IS_FP8_KV_CACHE,
609
+ vllm::Fp8KVCacheDataType KV_DTYPE,
610
int NUM_THREADS = 128>
611
void paged_attention_v1_launcher(
612
torch::Tensor& out,
@@ -706,36 +683,36 @@ void paged_attention_v1_launcher(
683
}
684
}
685
709
-#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
710
- paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE>( \
711
- out, \
712
- query, \
713
- key_cache, \
714
- value_cache, \
715
- num_kv_heads, \
716
- scale, \
717
- block_tables, \
718
- seq_lens, \
719
- max_seq_len, \
720
- alibi_slopes, \
686
+#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE) \
687
+ paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE>( \
688
+ out, \
689
+ query, \
690
+ key_cache, \
691
+ value_cache, \
692
+ num_kv_heads, \
693
+ scale, \
694
+ block_tables, \
695
+ seq_lens, \
696
+ max_seq_len, \
697
+ alibi_slopes, \
698
kv_scale);
699
700
// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
701
// 1, 2, 4, 64, 128, 256.
725
-#define CALL_V1_LAUNCHER_BLOCK_SIZE(T, CACHE_T, IS_FP8_KV_CACHE) \
726
- switch (block_size) { \
727
- case 8: \
728
- CALL_V1_LAUNCHER(T, CACHE_T, 8, IS_FP8_KV_CACHE); \
729
- break; \
730
- case 16: \
731
- CALL_V1_LAUNCHER(T, CACHE_T, 16, IS_FP8_KV_CACHE); \
732
- break; \
733
- case 32: \
734
- CALL_V1_LAUNCHER(T, CACHE_T, 32, IS_FP8_KV_CACHE); \
735
- break; \
736
- default: \
737
- TORCH_CHECK(false, "Unsupported block size: ", block_size); \
738
- break; \
702
+#define CALL_V1_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
703
+ switch (block_size) { \
704
+ case 8: \
705
+ CALL_V1_LAUNCHER(T, CACHE_T, 8, KV_DTYPE); \
706
+ break; \
707
+ case 16: \
708
+ CALL_V1_LAUNCHER(T, CACHE_T, 16, KV_DTYPE); \
709
+ break; \
710
+ case 32: \
711
+ CALL_V1_LAUNCHER(T, CACHE_T, 32, KV_DTYPE); \
712
+ break; \
713
+ default: \
714
+ TORCH_CHECK(false, "Unsupported block size: ", block_size); \
715
+ break; \
716
}
717
718
void paged_attention_v1(
@@ -752,65 +729,44 @@ void paged_attention_v1(
729
const c10::optional<torch::Tensor>& alibi_slopes,
730
const std::string& kv_cache_dtype,
731
float kv_scale) {
755
- if (kv_cache_dtype == "auto") {
756
- if (query.dtype() == at::ScalarType::Float) {
757
- CALL_V1_LAUNCHER_BLOCK_SIZE(float, float, false);
758
- } else if (query.dtype() == at::ScalarType::Half) {
759
- CALL_V1_LAUNCHER_BLOCK_SIZE(uint16_t, uint16_t, false);
760
- } else if (query.dtype() == at::ScalarType::BFloat16) {
761
- CALL_V1_LAUNCHER_BLOCK_SIZE(__nv_bfloat16, __nv_bfloat16, false);
762
- } else {
763
- TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
764
- }
765
- } else if (kv_cache_dtype == "fp8") {
766
- if (query.dtype() == at::ScalarType::Float) {
767
- CALL_V1_LAUNCHER_BLOCK_SIZE(float, uint8_t, true);
768
- } else if (query.dtype() == at::ScalarType::Half) {
769
- CALL_V1_LAUNCHER_BLOCK_SIZE(uint16_t, uint8_t, true);
770
- } else if (query.dtype() == at::ScalarType::BFloat16) {
771
- CALL_V1_LAUNCHER_BLOCK_SIZE(__nv_bfloat16, uint8_t, true);
772
- } else {
773
- TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
774
- }
775
- } else {
776
- TORCH_CHECK(false, "Unsupported data type of kv cache: ", kv_cache_dtype);
777
- }
732
+
733
+ DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype, CALL_V1_LAUNCHER_BLOCK_SIZE)
734
}
735
780
-#define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
781
- vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
782
- IS_FP8_KV_CACHE, PARTITION_SIZE> \
783
- <<<grid, block, shared_mem_size, stream>>>( \
784
- exp_sums_ptr, \
785
- max_logits_ptr, \
786
- tmp_out_ptr, \
787
- query_ptr, \
788
- key_cache_ptr, \
789
- value_cache_ptr, \
790
- num_kv_heads, \
791
- scale, \
792
- block_tables_ptr, \
793
- seq_lens_ptr, \
794
- max_num_blocks_per_seq, \
795
- alibi_slopes_ptr, \
796
- q_stride, \
797
- kv_block_stride, \
798
- kv_head_stride, \
799
- kv_scale); \
800
- vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, PARTITION_SIZE> \
801
- <<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
802
- out_ptr, \
803
- exp_sums_ptr, \
804
- max_logits_ptr, \
805
- tmp_out_ptr, \
806
- seq_lens_ptr, \
736
+#define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
737
+ vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
738
+ KV_DTYPE, PARTITION_SIZE> \
739
+ <<<grid, block, shared_mem_size, stream>>>( \
740
+ exp_sums_ptr, \
741
+ max_logits_ptr, \
742
+ tmp_out_ptr, \
743
+ query_ptr, \
744
+ key_cache_ptr, \
745
+ value_cache_ptr, \
746
+ num_kv_heads, \
747
+ scale, \
748
+ block_tables_ptr, \
749
+ seq_lens_ptr, \
750
+ max_num_blocks_per_seq, \
751
+ alibi_slopes_ptr, \
752
+ q_stride, \
753
+ kv_block_stride, \
754
+ kv_head_stride, \
755
+ kv_scale); \
756
+ vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, PARTITION_SIZE> \
757
+ <<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
758
+ out_ptr, \
759
+ exp_sums_ptr, \
760
+ max_logits_ptr, \
761
+ tmp_out_ptr, \
762
+ seq_lens_ptr, \
763
max_num_partitions);
764
765
template<
766
typename T,
767
typename CACHE_T,
768
int BLOCK_SIZE,
813
- bool IS_FP8_KV_CACHE,
769
+ vllm::Fp8KVCacheDataType KV_DTYPE,
770
int NUM_THREADS = 128,
771
int PARTITION_SIZE = 512>
772
void paged_attention_v2_launcher(
@@ -897,39 +853,39 @@ void paged_attention_v2_launcher(
853
}
854
}
855
900
-#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
901
- paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE>( \
902
- out, \
903
- exp_sums, \
904
- max_logits, \
905
- tmp_out, \
906
- query, \
907
- key_cache, \
908
- value_cache, \
909
- num_kv_heads, \
910
- scale, \
911
- block_tables, \
912
- seq_lens, \
913
- max_seq_len, \
914
- alibi_slopes, \
856
+#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE) \
857
+ paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE>( \
858
+ out, \
859
+ exp_sums, \
860
+ max_logits, \
861
+ tmp_out, \
862
+ query, \
863
+ key_cache, \
864
+ value_cache, \
865
+ num_kv_heads, \
866
+ scale, \
867
+ block_tables, \
868
+ seq_lens, \
869
+ max_seq_len, \
870
+ alibi_slopes, \
871
kv_scale);
872
873
// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
874
// 1, 2, 4, 64, 128, 256.
919
-#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, IS_FP8_KV_CACHE) \
920
- switch (block_size) { \
921
- case 8: \
922
- CALL_V2_LAUNCHER(T, CACHE_T, 8, IS_FP8_KV_CACHE); \
923
- break; \
924
- case 16: \
925
- CALL_V2_LAUNCHER(T, CACHE_T, 16, IS_FP8_KV_CACHE); \
926
- break; \
927
- case 32: \
928
- CALL_V2_LAUNCHER(T, CACHE_T, 32, IS_FP8_KV_CACHE); \
929
- break; \
930
- default: \
931
- TORCH_CHECK(false, "Unsupported block size: ", block_size); \
932
- break; \
875
+#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
876
+ switch (block_size) { \
877
+ case 8: \
878
+ CALL_V2_LAUNCHER(T, CACHE_T, 8, KV_DTYPE); \
879
+ break; \
880
+ case 16: \
881
+ CALL_V2_LAUNCHER(T, CACHE_T, 16, KV_DTYPE); \
882
+ break; \
883
+ case 32: \
884
+ CALL_V2_LAUNCHER(T, CACHE_T, 32, KV_DTYPE); \
885
+ break; \
886
+ default: \
887
+ TORCH_CHECK(false, "Unsupported block size: ", block_size); \
888
+ break; \
889
}
890
891
void paged_attention_v2(
@@ -949,29 +905,7 @@ void paged_attention_v2(
905
const c10::optional<torch::Tensor>& alibi_slopes,
906
const std::string& kv_cache_dtype,
907
float kv_scale) {
952
- if (kv_cache_dtype == "auto") {
953
- if (query.dtype() == at::ScalarType::Float) {
954
- CALL_V2_LAUNCHER_BLOCK_SIZE(float, float, false);
955
- } else if (query.dtype() == at::ScalarType::Half) {
956
- CALL_V2_LAUNCHER_BLOCK_SIZE(uint16_t, uint16_t, false);
957
- } else if (query.dtype() == at::ScalarType::BFloat16) {
958
- CALL_V2_LAUNCHER_BLOCK_SIZE(__nv_bfloat16, __nv_bfloat16, false);
959
- } else {
960
- TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
961
- }
962
- } else if (kv_cache_dtype == "fp8") {
963
- if (query.dtype() == at::ScalarType::Float) {
964
- CALL_V2_LAUNCHER_BLOCK_SIZE(float, uint8_t, true);
965
- } else if (query.dtype() == at::ScalarType::Half) {
966
- CALL_V2_LAUNCHER_BLOCK_SIZE(uint16_t, uint8_t, true);
967
- } else if (query.dtype() == at::ScalarType::BFloat16) {
968
- CALL_V2_LAUNCHER_BLOCK_SIZE(__nv_bfloat16, uint8_t, true);
969
- } else {
970
- TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
971
- }
972
- } else {
973
- TORCH_CHECK(false, "Unsupported data type of kv cache: ", kv_cache_dtype);
974
- }
908
+ DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype, CALL_V2_LAUNCHER_BLOCK_SIZE)
909
}
910
911
#undef WARP_SIZE
csrc/attention/dtype_fp8.cuh
+11
-5
@@ -3,14 +3,21 @@
3
#include "attention_generic.cuh"
4
5
#include <stdint.h>
6
-#ifdef ENABLE_FP8_E5M2
6
+#ifdef ENABLE_FP8
7
+#ifndef USE_ROCM
8
#include <cuda_fp8.h>
8
-#endif
9
+#endif // USE_ROCM
10
+#endif // ENABLE_FP8
11
12
namespace vllm {
11
-#if defined(ENABLE_FP8_E5M2) || defined(ENABLE_FP8_E4M3)
12
-// fp8 vector types for quantization of kv cache
13
14
+enum class Fp8KVCacheDataType {
15
+ kAuto = 0,
16
+ kFp8E4M3 = 1,
17
+ kFp8E5M2 = 2,
18
+};
19
+
20
+// fp8 vector types for quantization of kv cache
21
template<>
22
struct Vec<uint8_t, 1> {
23
using Type = uint8_t;
@@ -30,6 +37,5 @@ template<>
37
struct Vec<uint8_t, 8> {
38
using Type = uint2;
39
};
33
-#endif // ENABLE_FP8_E5M2
40
41
} // namespace vllm
csrc/cache.h
+3
-1
@@ -34,5 +34,7 @@ void reshape_and_cache_flash(
34
35
// Just for unittest
36
void convert_fp8(
37
+ torch::Tensor& dst_cache,
38
torch::Tensor& src_cache,
38
- torch::Tensor& dst_cache);
39
+ const float scale,
40
+ const std::string& kv_cache_dtype);
csrc/cache_kernels.cu
+70
-73
@@ -4,10 +4,11 @@
4
5
#include "cuda_compat.h"
6
#include "dispatch_utils.h"
7
-#if defined(ENABLE_FP8_E5M2)
8
-#include "quantization/fp8_e5m2_kvcache/quant_utils.cuh"
9
-#elif defined(ENABLE_FP8_E4M3)
10
-#include "quantization/fp8/amd_detail/quant_utils.cuh"
7
+
8
+#ifdef USE_ROCM
9
+#include "quantization/fp8/amd/quant_utils.cuh"
10
+#else
11
+#include "quantization/fp8/nvidia/quant_utils.cuh"
12
#endif
13
14
#include <algorithm>
@@ -149,7 +150,7 @@ void copy_blocks(
150
151
namespace vllm {
152
152
-template<typename scalar_t, typename cache_t, bool is_fp8_kv_cache>
153
+template<typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>
154
__global__ void reshape_and_cache_kernel(
155
const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
156
const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
@@ -194,19 +195,12 @@ __global__ void reshape_and_cache_kernel(
195
+ block_offset;
196
scalar_t tgt_key = key[src_key_idx];
197
scalar_t tgt_value = value[src_value_idx];
197
- if constexpr (is_fp8_kv_cache) {
198
-#if defined(ENABLE_FP8_E5M2)
199
- key_cache[tgt_key_idx] = fp8_e5m2_unscaled::vec_conversion<uint8_t, scalar_t>(tgt_key);
200
- value_cache[tgt_value_idx] = fp8_e5m2_unscaled::vec_conversion<uint8_t, scalar_t>(tgt_value);
201
-#elif defined(ENABLE_FP8_E4M3)
202
- key_cache[tgt_key_idx] = fp8_e4m3::scaled_vec_conversion<uint8_t, scalar_t>(tgt_key, kv_scale);
203
- value_cache[tgt_value_idx] = fp8_e4m3::scaled_vec_conversion<uint8_t, scalar_t>(tgt_value, kv_scale);
204
-#else
205
- assert(false);
206
-#endif
207
- } else {
198
+ if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) {
199
key_cache[tgt_key_idx] = tgt_key;
200
value_cache[tgt_value_idx] = tgt_value;
201
+ } else {
202
+ key_cache[tgt_key_idx] = fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, kv_scale);
203
+ value_cache[tgt_value_idx] = fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, kv_scale);
204
}
205
}
206
}
@@ -248,19 +242,22 @@ __global__ void reshape_and_cache_flash_kernel(
242
}
243
} // namespace vllm
244
251
-#define CALL_RESHAPE_AND_CACHE(KV_T, CACHE_T, IS_FP8_KV_CACHE) \
252
- vllm::reshape_and_cache_kernel<KV_T, CACHE_T, IS_FP8_KV_CACHE><<<grid, block, 0, stream>>>( \
253
- reinterpret_cast<KV_T*>(key.data_ptr()), \
254
- reinterpret_cast<KV_T*>(value.data_ptr()), \
255
- reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
256
- reinterpret_cast<CACHE_T*>(value_cache.data_ptr()), \
257
- slot_mapping.data_ptr<int64_t>(), \
258
- key_stride, \
259
- value_stride, \
260
- num_heads, \
261
- head_size, \
262
- block_size, \
263
- x, \
245
+// KV_T is the stored data type of kv-cache.
246
+// CACHE_T is the data type of key and value tensors.
247
+// KV_DTYPE is the real data type of kv-cache.
248
+#define CALL_RESHAPE_AND_CACHE(KV_T, CACHE_T, KV_DTYPE) \
249
+ vllm::reshape_and_cache_kernel<KV_T, CACHE_T, KV_DTYPE><<<grid, block, 0, stream>>>( \
250
+ reinterpret_cast<KV_T*>(key.data_ptr()), \
251
+ reinterpret_cast<KV_T*>(value.data_ptr()), \
252
+ reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
253
+ reinterpret_cast<CACHE_T*>(value_cache.data_ptr()), \
254
+ slot_mapping.data_ptr<int64_t>(), \
255
+ key_stride, \
256
+ value_stride, \
257
+ num_heads, \
258
+ head_size, \
259
+ block_size, \
260
+ x, \
261
kv_scale);
262
263
void reshape_and_cache(
@@ -285,25 +282,8 @@ void reshape_and_cache(
282
dim3 block(std::min(num_heads * head_size, 512));
283
const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
284
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
288
- if (kv_cache_dtype == "auto") {
289
- if (key.dtype() == at::ScalarType::Float) {
290
- CALL_RESHAPE_AND_CACHE(float, float, false);
291
- } else if (key.dtype() == at::ScalarType::Half) {
292
- CALL_RESHAPE_AND_CACHE(uint16_t, uint16_t, false);
293
- } else if (key.dtype() == at::ScalarType::BFloat16) {
294
- CALL_RESHAPE_AND_CACHE(__nv_bfloat16, __nv_bfloat16, false);
295
- }
296
- } else if (kv_cache_dtype == "fp8") {
297
- if (key.dtype() == at::ScalarType::Float) {
298
- CALL_RESHAPE_AND_CACHE(float, uint8_t, true);
299
- } else if (key.dtype() == at::ScalarType::Half) {
300
- CALL_RESHAPE_AND_CACHE(uint16_t, uint8_t, true);
301
- } else if (key.dtype() == at::ScalarType::BFloat16) {
302
- CALL_RESHAPE_AND_CACHE(__nv_bfloat16, uint8_t, true);
303
- }
304
- } else {
305
- TORCH_CHECK(false, "Unsupported data type of kv cache: ", kv_cache_dtype);
306
- }
285
+
286
+ DISPATCH_BY_KV_CACHE_DTYPE(key.dtype(), kv_cache_dtype, CALL_RESHAPE_AND_CACHE)
287
}
288
289
void reshape_and_cache_flash(
@@ -353,35 +333,34 @@ void reshape_and_cache_flash(
333
334
namespace vllm {
335
356
-template<typename Tout, typename Tin>
336
+template<typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
337
__global__ void convert_fp8_kernel(
338
const Tin* __restrict__ src_cache,
339
Tout* __restrict__ dst_cache,
340
+ const float kv_scale,
341
const int64_t block_stride) {
342
const int64_t block_idx = blockIdx.x;
343
for (int i = threadIdx.x; i < block_stride; i += blockDim.x) {
344
int64_t idx = block_idx * block_stride + i;
364
-#if defined(ENABLE_FP8_E5M2)
365
- dst_cache[idx] = fp8_e5m2_unscaled::vec_conversion<Tout, Tin>(src_cache[idx]);
366
-#elif defined(ENABLE_FP8_E4M3)
367
- dst_cache[idx] = fp8_e4m3::vec_conversion<Tout, Tin>(src_cache[idx]);
368
-#else
369
- assert(false);
370
-#endif
345
+ dst_cache[idx] = fp8::scaled_convert<Tout, Tin, kv_dt>(src_cache[idx], kv_scale);
346
}
347
}
348
349
} // namespace vllm
350
376
-#define CALL_CONVERT_FP8(Tout, Tin) \
377
- vllm::convert_fp8_kernel<Tout, Tin><<<grid, block, 0, stream>>>( \
378
- reinterpret_cast<Tin*>(src_cache.data_ptr()), \
379
- reinterpret_cast<Tout*>(dst_cache.data_ptr()), \
351
+#define CALL_CONVERT_FP8(Tout, Tin, KV_DTYPE) \
352
+ vllm::convert_fp8_kernel<Tout, Tin, KV_DTYPE><<<grid, block, 0, stream>>>( \
353
+ reinterpret_cast<Tin*>(src_cache.data_ptr()), \
354
+ reinterpret_cast<Tout*>(dst_cache.data_ptr()), \
355
+ kv_scale, \
356
block_stride);
357
358
+// Only for testing.
359
void convert_fp8(
360
+ torch::Tensor& dst_cache,
361
torch::Tensor& src_cache,
384
- torch::Tensor& dst_cache)
362
+ const float kv_scale,
363
+ const std::string& kv_cache_dtype)
364
{
365
torch::Device src_device = src_cache.device();
366
torch::Device dst_device = dst_cache.device();
@@ -399,17 +378,35 @@ void convert_fp8(
378
dim3 block(std::min(block_stride, int64_t(512)));
379
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
380
402
- if (src_cache.dtype() == at::ScalarType::Float) {
403
- CALL_CONVERT_FP8(uint8_t, float);
404
- } else if (src_cache.dtype() == at::ScalarType::Half) {
405
- CALL_CONVERT_FP8(uint8_t, uint16_t);
406
- } else if (src_cache.dtype() == at::ScalarType::BFloat16) {
407
- CALL_CONVERT_FP8(uint8_t, __nv_bfloat16);
408
- } else if (dst_cache.dtype() == at::ScalarType::Float) {
409
- CALL_CONVERT_FP8(float, uint8_t);
410
- } else if (dst_cache.dtype() == at::ScalarType::Half) {
411
- CALL_CONVERT_FP8(uint16_t, uint8_t);
412
- } else if (dst_cache.dtype() == at::ScalarType::BFloat16) {
413
- CALL_CONVERT_FP8(__nv_bfloat16, uint8_t);
381
+ if (kv_cache_dtype == "auto") {
382
+ if (src_cache.dtype() == at::ScalarType::Float) {
383
+ CALL_CONVERT_FP8(uint8_t, float, vllm::Fp8KVCacheDataType::kAuto);
384
+ } else if (src_cache.dtype() == at::ScalarType::Half) {
385
+ CALL_CONVERT_FP8(uint8_t, uint16_t, vllm::Fp8KVCacheDataType::kAuto);
386
+ } else if (src_cache.dtype() == at::ScalarType::BFloat16) {
387
+ CALL_CONVERT_FP8(uint8_t, __nv_bfloat16, vllm::Fp8KVCacheDataType::kAuto);
388
+ } else if (dst_cache.dtype() == at::ScalarType::Float) {
389
+ CALL_CONVERT_FP8(float, uint8_t, vllm::Fp8KVCacheDataType::kAuto);
390
+ } else if (dst_cache.dtype() == at::ScalarType::Half) {
391
+ CALL_CONVERT_FP8(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kAuto);
392
+ } else if (dst_cache.dtype() == at::ScalarType::BFloat16) {
393
+ CALL_CONVERT_FP8(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kAuto);
394
+ }
395
+ } else if (kv_cache_dtype == "fp8" || kv_cache_dtype == "fp8_e4m3") {
396
+ if (src_cache.dtype() == at::ScalarType::Float) {
397
+ CALL_CONVERT_FP8(uint8_t, float, vllm::Fp8KVCacheDataType::kFp8E4M3);
398
+ } else if (src_cache.dtype() == at::ScalarType::Half) {
399
+ CALL_CONVERT_FP8(uint8_t, uint16_t, vllm::Fp8KVCacheDataType::kFp8E4M3);
400
+ } else if (src_cache.dtype() == at::ScalarType::BFloat16) {
401
+ CALL_CONVERT_FP8(uint8_t, __nv_bfloat16, vllm::Fp8KVCacheDataType::kFp8E4M3);
402
+ } else if (dst_cache.dtype() == at::ScalarType::Float) {
403
+ CALL_CONVERT_FP8(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3);
404
+ } else if (dst_cache.dtype() == at::ScalarType::Half) {
405
+ CALL_CONVERT_FP8(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3);
406
+ } else if (dst_cache.dtype() == at::ScalarType::BFloat16) {
407
+ CALL_CONVERT_FP8(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3);
408
+ }
409
+ } else {
410
+ TORCH_CHECK(false, "Unsupported data type: ", kv_cache_dtype);
411
}
412
}
csrc/quantization/fp8/amd/hip_float8.h
renamed
csrc/quantization/fp8/amd/hip_float8_impl.h
renamed
csrc/quantization/fp8/amd/quant_utils.cuh
renamed
+58
-1
@@ -5,12 +5,17 @@
5
#include <hip/hip_bf16.h>
6
#include <hip/hip_bfloat16.h>
7
8
+#include "../../../attention/dtype_fp8.cuh"
9
#include "../../../attention/dtype_float32.cuh"
10
#include "../../../attention/dtype_bfloat16.cuh"
11
12
namespace vllm
13
{
13
-namespace fp8_e4m3 {
14
+#ifdef USE_ROCM
15
+
16
+namespace fp8 {
17
+#ifdef ENABLE_FP8
18
+
19
template <typename Tout, typename Tin>
20
__inline__ __device__ Tout vec_conversion(const Tin& x)
21
{
@@ -512,6 +517,58 @@ __inline__ __device__ float4 scaled_vec_conversion<float4, uint32_t>(const uint3
517
float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
518
return res;
519
}
520
+#endif // ENABLE_FP8
521
522
+template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
523
+__inline__ __device__ Tout convert(const Tin &x) {
524
+#ifdef ENABLE_FP8
525
+ if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E4M3) {
526
+ return vec_conversion<Tout, Tin>(x);
527
+ }
528
+#endif
529
+ assert(false);
530
}
531
+
532
+template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
533
+__inline__ __device__ Tout scaled_convert(const Tin &x, const float scale) {
534
+#ifdef ENABLE_FP8
535
+ if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E4M3) {
536
+ return scaled_vec_conversion<Tout, Tin>(x, scale);
537
+ }
538
+#endif
539
+ assert(false);
540
+}
541
+
542
+// The following macro is used to dispatch the conversion function based on the
543
+// data type of the key and value cache. The FN is a macro that calls a function
544
+// with template<typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>.
545
+#define DISPATCH_BY_KV_CACHE_DTYPE(SRC_DTYPE, KV_DTYPE, FN) \
546
+ if (KV_DTYPE == "auto") { \
547
+ if (SRC_DTYPE == at::ScalarType::Float) { \
548
+ FN(float, float, vllm::Fp8KVCacheDataType::kAuto); \
549
+ } else if (SRC_DTYPE == at::ScalarType::Half) { \
550
+ FN(uint16_t, uint16_t, vllm::Fp8KVCacheDataType::kAuto); \
551
+ } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
552
+ FN(__nv_bfloat16, __nv_bfloat16, vllm::Fp8KVCacheDataType::kAuto); \
553
+ } else { \
554
+ TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
555
+ } \
556
+ } else { \
557
+ if (KV_DTYPE == "fp8" || KV_DTYPE == "fp8_e4m3") { \
558
+ if (SRC_DTYPE == at::ScalarType::Float) { \
559
+ FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
560
+ } else if (SRC_DTYPE == at::ScalarType::Half) { \
561
+ FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
562
+ } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
563
+ FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
564
+ } else { \
565
+ TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
566
+ } \
567
+ } else { \
568
+ TORCH_CHECK(false, "Unsupported data type of kv cache: ", KV_DTYPE); \
569
+ } \
570
+ }
571
+
572
+} // fp8
573
+#endif // USE_ROCM
574
} // namespace vllm
csrc/quantization/fp8/common.cu
renamed
csrc/quantization/fp8/nvidia/quant_utils.cuh
new
+568
@@ -0,0 +1,568 @@
1
+#pragma once
2
+
3
+#include "../../../attention/attention_dtypes.h"
4
+#include <assert.h>
5
+#include <float.h>
6
+#include <stdint.h>
7
+#include <type_traits>
8
+
9
+namespace vllm {
10
+#ifndef USE_ROCM
11
+
12
+namespace fp8 {
13
+#ifdef ENABLE_FP8
14
+
15
+#if 0 // Disable the following code to reduce the binary size.
16
+template <typename Tout, typename Tin>
17
+__inline__ __device__ Tout
18
+vec_conversion(const Tin &x, const __nv_fp8_interpretation_t fp8_type) {
19
+ return x;
20
+}
21
+
22
+// fp8 -> half
23
+template <>
24
+__inline__ __device__ uint16_t vec_conversion<uint16_t, uint8_t>(
25
+ const uint8_t &a, const __nv_fp8_interpretation_t fp8_type) {
26
+ __half_raw res = __nv_cvt_fp8_to_halfraw(a, fp8_type);
27
+ return res.x;
28
+}
29
+
30
+// fp8x2 -> half2
31
+template <>
32
+__inline__ __device__ uint32_t vec_conversion<uint32_t, uint16_t>(
33
+ const uint16_t &a, const __nv_fp8_interpretation_t fp8_type) {
34
+ union {
35
+ uint16_t u16[2];
36
+ uint32_t u32;
37
+ } tmp;
38
+ __half2_raw res = __nv_cvt_fp8x2_to_halfraw2(a, fp8_type);
39
+ tmp.u16[0] = res.x;
40
+ tmp.u16[1] = res.y;
41
+ return tmp.u32;
42
+}
43
+
44
+// fp8x4 -> half2x2
45
+template <>
46
+__inline__ __device__ uint2 vec_conversion<uint2, uint32_t>(
47
+ const uint32_t &a, const __nv_fp8_interpretation_t fp8_type) {
48
+ union {
49
+ uint2 u32x2;
50
+ uint32_t u32[2];
51
+ } tmp;
52
+ tmp.u32[0] = vec_conversion<uint32_t, uint16_t>((uint16_t)a, fp8_type);
53
+ tmp.u32[1] =
54
+ vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U), fp8_type);
55
+ return tmp.u32x2;
56
+}
57
+
58
+// fp8x8 -> half2x4
59
+template <>
60
+__inline__ __device__ uint4 vec_conversion<uint4, uint2>(
61
+ const uint2 &a, const __nv_fp8_interpretation_t fp8_type) {
62
+ union {
63
+ uint4 u64x2;
64
+ uint2 u64[2];
65
+ } tmp;
66
+ tmp.u64[0] = vec_conversion<uint2, uint32_t>(a.x, fp8_type);
67
+ tmp.u64[1] = vec_conversion<uint2, uint32_t>(a.y, fp8_type);
68
+ return tmp.u64x2;
69
+}
70
+
71
+// fp8 -> __nv_bfloat16
72
+template <>
73
+__inline__ __device__ __nv_bfloat16 vec_conversion<__nv_bfloat16, uint8_t>(
74
+ const uint8_t &a, const __nv_fp8_interpretation_t fp8_type) {
75
+ // Note there is no direct convert function from fp8 to bf16.
76
+ // fp8 -> half
77
+ __half_raw res = __nv_cvt_fp8_to_halfraw(a, fp8_type);
78
+ // half -> float -> bf16
79
+ float tmp = half_to_float(res.x);
80
+ return __float2bfloat16(tmp);
81
+}
82
+
83
+// fp8x2 -> __nv_bfloat162
84
+template <>
85
+__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, uint16_t>(
86
+ const uint16_t &a, const __nv_fp8_interpretation_t fp8_type) {
87
+ __nv_bfloat162 res;
88
+ res.x = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a, fp8_type);
89
+ res.y = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U), fp8_type);
90
+ return res;
91
+}
92
+
93
+// fp8x4 -> bf16_4_t
94
+template <>
95
+__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, uint32_t>(
96
+ const uint32_t &a, const __nv_fp8_interpretation_t fp8_type) {
97
+ bf16_4_t res;
98
+ res.x = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a, fp8_type);
99
+ res.y =
100
+ vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U), fp8_type);
101
+ return res;
102
+}
103
+
104
+// fp8x8 -> bf16_8_t
105
+template <>
106
+__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, uint2>(
107
+ const uint2 &a, const __nv_fp8_interpretation_t fp8_type) {
108
+ bf16_4_t tmp1, tmp2;
109
+ tmp1 = vec_conversion<bf16_4_t, uint32_t>(a.x, fp8_type);
110
+ tmp2 = vec_conversion<bf16_4_t, uint32_t>(a.y, fp8_type);
111
+ bf16_8_t res;
112
+ res.x = tmp1.x;
113
+ res.y = tmp1.y;
114
+ res.z = tmp2.x;
115
+ res.w = tmp2.y;
116
+ return res;
117
+}
118
+
119
+// fp8 -> float
120
+template <>
121
+__inline__ __device__ float
122
+vec_conversion<float, uint8_t>(const uint8_t &a,
123
+ const __nv_fp8_interpretation_t fp8_type) {
124
+ // fp8 -> half
125
+ uint16_t tmp = vec_conversion<uint16_t, uint8_t>(a, fp8_type);
126
+ // half -> float
127
+ return half_to_float(tmp);
128
+}
129
+
130
+// fp8x2 -> float2
131
+template <>
132
+__inline__ __device__ float2 vec_conversion<float2, uint16_t>(
133
+ const uint16_t &a, const __nv_fp8_interpretation_t fp8_type) {
134
+ // fp8x2 -> half2
135
+ uint32_t tmp = vec_conversion<uint32_t, uint16_t>(a, fp8_type);
136
+ // half2 -> float2
137
+ return half2_to_float2(tmp);
138
+}
139
+
140
+// fp8x4 -> float4
141
+template <>
142
+__inline__ __device__ Float4_ vec_conversion<Float4_, uint32_t>(
143
+ const uint32_t &a, const __nv_fp8_interpretation_t fp8_type) {
144
+ Float4_ res;
145
+ res.x = vec_conversion<float2, uint16_t>((uint16_t)a, fp8_type);
146
+ res.y = vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U), fp8_type);
147
+ return res;
148
+}
149
+
150
+// fp8x8 -> float8
151
+template <>
152
+__inline__ __device__ Float8_ vec_conversion<Float8_, uint2>(
153
+ const uint2 &a, const __nv_fp8_interpretation_t fp8_type) {
154
+ Float4_ tmp1, tmp2;
155
+ tmp1 = vec_conversion<Float4_, uint32_t>(a.x, fp8_type);
156
+ tmp2 = vec_conversion<Float4_, uint32_t>(a.y, fp8_type);
157
+ Float8_ res;
158
+ res.x = tmp1.x;
159
+ res.y = tmp1.y;
160
+ res.z = tmp2.x;
161
+ res.w = tmp2.y;
162
+ return res;
163
+}
164
+
165
+// half -> fp8
166
+template <>
167
+__inline__ __device__ uint8_t vec_conversion<uint8_t, uint16_t>(
168
+ const uint16_t &a, const __nv_fp8_interpretation_t fp8_type) {
169
+ __half_raw tmp;
170
+ tmp.x = a;
171
+ __nv_fp8_storage_t res =
172
+ __nv_cvt_halfraw_to_fp8(tmp, __NV_SATFINITE, fp8_type);
173
+ return (uint8_t)res;
174
+}
175
+
176
+// bf16 -> fp8
177
+template <>
178
+__inline__ __device__ uint8_t vec_conversion<uint8_t, __nv_bfloat16>(
179
+ const __nv_bfloat16 &a, const __nv_fp8_interpretation_t fp8_type) {
180
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
181
+ assert(false);
182
+#else
183
+ __nv_fp8_storage_t res = __nv_cvt_bfloat16raw_to_fp8(
184
+ __nv_bfloat16_raw(a), __NV_SATFINITE, fp8_type);
185
+ return (uint8_t)res;
186
+#endif
187
+}
188
+
189
+// float -> fp8
190
+template <>
191
+__inline__ __device__ uint8_t vec_conversion<uint8_t, float>(
192
+ const float &a, const __nv_fp8_interpretation_t fp8_type) {
193
+ __nv_fp8_storage_t res = __nv_cvt_float_to_fp8(a, __NV_SATFINITE, fp8_type);
194
+ return (uint8_t)res;
195
+}
196
+
197
+// fp8x4 -> float4
198
+template <>
199
+__inline__ __device__ float4 vec_conversion<float4, uint32_t>(
200
+ const uint32_t &a, const __nv_fp8_interpretation_t fp8_type) {
201
+ Float4_ tmp = vec_conversion<Float4_, uint32_t>(a, fp8_type);
202
+ float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
203
+ return res;
204
+}
205
+
206
+template <>
207
+__inline__ __device__ uint32_t vec_conversion<uint32_t, float2>(
208
+ const float2 &a, const __nv_fp8_interpretation_t fp8_type) {
209
+ union {
210
+ half2 float16;
211
+ uint32_t uint32;
212
+ };
213
+
214
+ float16 = __float22half2_rn(a);
215
+ return uint32;
216
+}
217
+
218
+template <>
219
+__inline__ __device__ uint2 vec_conversion<uint2, Float4_>(
220
+ const Float4_ &a, const __nv_fp8_interpretation_t fp8_type) {
221
+ uint2 b;
222
+ float2 val;
223
+ val.x = a.x.x;
224
+ val.y = a.x.y;
225
+ b.x = vec_conversion<uint32_t, float2>(val, fp8_type);
226
+
227
+ val.x = a.y.x;
228
+ val.y = a.y.y;
229
+ b.y = vec_conversion<uint32_t, float2>(val, fp8_type);
230
+
231
+ return b;
232
+}
233
+
234
+template <>
235
+__inline__ __device__ float4 vec_conversion<float4, Float4_>(
236
+ const Float4_ &a, const __nv_fp8_interpretation_t fp8_type) {
237
+ float4 b;
238
+ b.x = a.x.x;
239
+ b.y = a.x.y;
240
+ b.z = a.y.x;
241
+ b.w = a.y.y;
242
+ return b;
243
+}
244
+
245
+template <>
246
+__inline__ __device__ uint4 vec_conversion<uint4, Float8_>(
247
+ const Float8_ &a, const __nv_fp8_interpretation_t fp8_type) {
248
+ uint4 b;
249
+ b.x = vec_conversion<uint32_t, float2>(a.x, fp8_type);
250
+ b.y = vec_conversion<uint32_t, float2>(a.y, fp8_type);
251
+ b.z = vec_conversion<uint32_t, float2>(a.z, fp8_type);
252
+ b.w = vec_conversion<uint32_t, float2>(a.w, fp8_type);
253
+ return b;
254
+}
255
+
256
+template <>
257
+__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, float2>(
258
+ const float2 &a, const __nv_fp8_interpretation_t fp8_type) {
259
+ __nv_bfloat162 b;
260
+ from_float(b, a);
261
+ return b;
262
+}
263
+
264
+template <>
265
+__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, Float4_>(
266
+ const Float4_ &a, const __nv_fp8_interpretation_t fp8_type) {
267
+ bf16_4_t b;
268
+ from_float(b, a);
269
+ return b;
270
+}
271
+
272
+template <>
273
+__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, Float8_>(
274
+ const Float8_ &a, const __nv_fp8_interpretation_t fp8_type) {
275
+ bf16_8_t b;
276
+ from_float(b, a);
277
+ return b;
278
+}
279
+#endif
280
+
281
+/* Scaled and vectorized conversions, for data exchange between high and low
282
+ precision domains Convention of the scale in API, e.g: FP8_data =
283
+ Quantization( High_Precision_data / scale ) s.t. Quantize(HP / scale) => FP8
284
+ Dequant(FP8) * scale => HP
285
+ */
286
+
287
+template <typename Tout, typename Tin>
288
+__inline__ __device__ Tout scaled_vec_conversion(
289
+ const Tin &x, const float scale, const __nv_fp8_interpretation_t fp8_type) {
290
+ return x;
291
+}
292
+
293
+// fp8 -> half
294
+template <>
295
+__inline__ __device__ uint16_t scaled_vec_conversion<uint16_t, uint8_t>(
296
+ const uint8_t &a, const float scale,
297
+ const __nv_fp8_interpretation_t fp8_type) {
298
+ __half_raw tmp = __nv_cvt_fp8_to_halfraw(a, fp8_type);
299
+ return float_to_half(half_to_float(tmp.x) * scale);
300
+}
301
+
302
+// fp8x2 -> half2
303
+template <>
304
+__inline__ __device__ uint32_t scaled_vec_conversion<uint32_t, uint16_t>(
305
+ const uint16_t &a, const float scale,
306
+ const __nv_fp8_interpretation_t fp8_type) {
307
+ union {
308
+ uint16_t u16[2];
309
+ uint32_t u32;
310
+ } tmp;
311
+ __half2_raw res = __nv_cvt_fp8x2_to_halfraw2(a, fp8_type);
312
+ tmp.u16[0] = float_to_half(half_to_float(res.x) * scale);
313
+ tmp.u16[1] = float_to_half(half_to_float(res.y) * scale);
314
+ return tmp.u32;
315
+}
316
+
317
+// fp8x4 -> half2x2
318
+template <>
319
+__inline__ __device__ uint2 scaled_vec_conversion<uint2, uint32_t>(
320
+ const uint32_t &a, const float scale,
321
+ const __nv_fp8_interpretation_t fp8_type) {
322
+ union {
323
+ uint2 u32x2;
324
+ uint32_t u32[2];
325
+ } tmp;
326
+ tmp.u32[0] =
327
+ scaled_vec_conversion<uint32_t, uint16_t>((uint16_t)a, scale, fp8_type);
328
+ tmp.u32[1] = scaled_vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U),
329
+ scale, fp8_type);
330
+ return tmp.u32x2;
331
+}
332
+
333
+// fp8x8 -> half2x4
334
+template <>
335
+__inline__ __device__ uint4
336
+scaled_vec_conversion<uint4, uint2>(const uint2 &a, const float scale,
337
+ const __nv_fp8_interpretation_t fp8_type) {
338
+ union {
339
+ uint4 u64x2;
340
+ uint2 u64[2];
341
+ } tmp;
342
+ tmp.u64[0] = scaled_vec_conversion<uint2, uint32_t>(a.x, scale, fp8_type);
343
+ tmp.u64[1] = scaled_vec_conversion<uint2, uint32_t>(a.y, scale, fp8_type);
344
+ return tmp.u64x2;
345
+}
346
+
347
+// fp8 -> __nv_bfloat16
348
+template <>
349
+__inline__ __device__ __nv_bfloat16
350
+scaled_vec_conversion<__nv_bfloat16, uint8_t>(
351
+ const uint8_t &a, const float scale,
352
+ const __nv_fp8_interpretation_t fp8_type) {
353
+ // Note there is no direct convert function from fp8 to bf16.
354
+ // fp8 -> half
355
+ __half_raw res = __nv_cvt_fp8_to_halfraw(a, fp8_type);
356
+ // half -> float -> bf16
357
+ float tmp = half_to_float(res.x);
358
+ return __float2bfloat16(tmp * scale);
359
+}
360
+
361
+// fp8x2 -> __nv_bfloat162
362
+template <>
363
+__inline__ __device__ __nv_bfloat162
364
+scaled_vec_conversion<__nv_bfloat162, uint16_t>(
365
+ const uint16_t &a, const float scale,
366
+ const __nv_fp8_interpretation_t fp8_type) {
367
+ __nv_bfloat162 res;
368
+ res.x = scaled_vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a, scale,
369
+ fp8_type);
370
+ res.y = scaled_vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U),
371
+ scale, fp8_type);
372
+ return res;
373
+}
374
+
375
+// fp8x4 -> bf16_4_t
376
+template <>
377
+__inline__ __device__ bf16_4_t scaled_vec_conversion<bf16_4_t, uint32_t>(
378
+ const uint32_t &a, const float scale,
379
+ const __nv_fp8_interpretation_t fp8_type) {
380
+ bf16_4_t res;
381
+ res.x = scaled_vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a, scale,
382
+ fp8_type);
383
+ res.y = scaled_vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U),
384
+ scale, fp8_type);
385
+ return res;
386
+}
387
+
388
+// fp8x8 -> bf16_8_t
389
+template <>
390
+__inline__ __device__ bf16_8_t scaled_vec_conversion<bf16_8_t, uint2>(
391
+ const uint2 &a, const float scale,
392
+ const __nv_fp8_interpretation_t fp8_type) {
393
+ bf16_4_t tmp1, tmp2;
394
+ tmp1 = scaled_vec_conversion<bf16_4_t, uint32_t>(a.x, scale, fp8_type);
395
+ tmp2 = scaled_vec_conversion<bf16_4_t, uint32_t>(a.y, scale, fp8_type);
396
+ bf16_8_t res;
397
+ res.x = tmp1.x;
398
+ res.y = tmp1.y;
399
+ res.z = tmp2.x;
400
+ res.w = tmp2.y;
401
+ return res;
402
+}
403
+
404
+// fp8 -> float
405
+template <>
406
+__inline__ __device__ float scaled_vec_conversion<float, uint8_t>(
407
+ const uint8_t &a, const float scale,
408
+ const __nv_fp8_interpretation_t fp8_type) {
409
+
410
+ // fp8 -> half
411
+ __half_raw res = __nv_cvt_fp8_to_halfraw(a, fp8_type);
412
+ uint16_t tmp = res.x;
413
+
414
+ // half -> float
415
+ return half_to_float(tmp) * scale;
416
+}
417
+
418
+// fp8x2 -> float2
419
+template <>
420
+__inline__ __device__ float2 scaled_vec_conversion<float2, uint16_t>(
421
+ const uint16_t &a, const float scale,
422
+ const __nv_fp8_interpretation_t fp8_type) {
423
+ // fp8x2 -> half2
424
+ uint32_t tmp = scaled_vec_conversion<uint32_t, uint16_t>(a, scale, fp8_type);
425
+ // half2 -> float2
426
+ return half2_to_float2(tmp);
427
+}
428
+
429
+// fp8x4 -> float4
430
+template <>
431
+__inline__ __device__ Float4_ scaled_vec_conversion<Float4_, uint32_t>(
432
+ const uint32_t &a, const float scale,
433
+ const __nv_fp8_interpretation_t fp8_type) {
434
+ Float4_ res;
435
+ res.x = scaled_vec_conversion<float2, uint16_t>((uint16_t)a, scale, fp8_type);
436
+ res.y = scaled_vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U), scale,
437
+ fp8_type);
438
+ return res;
439
+}
440
+
441
+// fp8x8 -> float8
442
+template <>
443
+__inline__ __device__ Float8_ scaled_vec_conversion<Float8_, uint2>(
444
+ const uint2 &a, const float scale,
445
+ const __nv_fp8_interpretation_t fp8_type) {
446
+ Float4_ tmp1, tmp2;
447
+ tmp1 = scaled_vec_conversion<Float4_, uint32_t>(a.x, scale, fp8_type);
448
+ tmp2 = scaled_vec_conversion<Float4_, uint32_t>(a.y, scale, fp8_type);
449
+ Float8_ res;
450
+ res.x = tmp1.x;
451
+ res.y = tmp1.y;
452
+ res.z = tmp2.x;
453
+ res.w = tmp2.y;
454
+ return res;
455
+}
456
+
457
+// half -> fp8
458
+template <>
459
+__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, uint16_t>(
460
+ const uint16_t &a, const float scale,
461
+ const __nv_fp8_interpretation_t fp8_type) {
462
+ __nv_fp8_storage_t res =
463
+ __nv_cvt_float_to_fp8(half_to_float(a) / scale, __NV_SATFINITE, fp8_type);
464
+ return (uint8_t)res;
465
+}
466
+
467
+// bf16 -> fp8
468
+template <>
469
+__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, __nv_bfloat16>(
470
+ const __nv_bfloat16 &a, const float scale,
471
+ const __nv_fp8_interpretation_t fp8_type) {
472
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
473
+ assert(false);
474
+#else
475
+ __nv_fp8_storage_t res = __nv_cvt_float_to_fp8(__bfloat162float(a) / scale,
476
+ __NV_SATFINITE, fp8_type);
477
+ return (uint8_t)res;
478
+#endif
479
+}
480
+
481
+// float -> fp8
482
+template <>
483
+__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, float>(
484
+ const float &a, const float scale,
485
+ const __nv_fp8_interpretation_t fp8_type) {
486
+ __nv_fp8_storage_t res =
487
+ __nv_cvt_float_to_fp8(a / scale, __NV_SATFINITE, fp8_type);
488
+ return (uint8_t)res;
489
+}
490
+
491
+// fp8x4 -> float4
492
+template <>
493
+__inline__ __device__ float4 scaled_vec_conversion<float4, uint32_t>(
494
+ const uint32_t &a, const float scale,
495
+ const __nv_fp8_interpretation_t fp8_type) {
496
+ Float4_ tmp = scaled_vec_conversion<Float4_, uint32_t>(a, scale, fp8_type);
497
+ float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
498
+ return res;
499
+}
500
+#endif // ENABLE_FP8
501
+
502
+template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
503
+__inline__ __device__ Tout convert(const Tin &x) {
504
+#if 0 // Disable the following code to reduce the binary size.
505
+ if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E4M3) {
506
+ return vec_conversion<Tout, Tin>(x, __NV_E4M3);
507
+ } else if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E5M2) {
508
+ return vec_conversion<Tout, Tin>(x, __NV_E5M2);
509
+ }
510
+#endif
511
+ assert(false);
512
+}
513
+
514
+template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
515
+__inline__ __device__ Tout scaled_convert(const Tin &x, const float scale) {
516
+#ifdef ENABLE_FP8
517
+ if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E4M3) {
518
+ return scaled_vec_conversion<Tout, Tin>(x, scale, __NV_E4M3);
519
+ } else if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E5M2) {
520
+ return scaled_vec_conversion<Tout, Tin>(x, scale, __NV_E5M2);
521
+ }
522
+#endif
523
+ assert(false);
524
+}
525
+
526
+// The following macro is used to dispatch the conversion function based on the
527
+// data type of the key and value cache. The FN is a macro that calls a function
528
+// with template<typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>.
529
+#define DISPATCH_BY_KV_CACHE_DTYPE(SRC_DTYPE, KV_DTYPE, FN) \
530
+ if (KV_DTYPE == "auto") { \
531
+ if (SRC_DTYPE == at::ScalarType::Float) { \
532
+ FN(float, float, vllm::Fp8KVCacheDataType::kAuto); \
533
+ } else if (SRC_DTYPE == at::ScalarType::Half) { \
534
+ FN(uint16_t, uint16_t, vllm::Fp8KVCacheDataType::kAuto); \
535
+ } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
536
+ FN(__nv_bfloat16, __nv_bfloat16, vllm::Fp8KVCacheDataType::kAuto); \
537
+ } else { \
538
+ TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
539
+ } \
540
+ } else { \
541
+ if (KV_DTYPE == "fp8" || KV_DTYPE == "fp8_e4m3") { \
542
+ if (SRC_DTYPE == at::ScalarType::Float) { \
543
+ FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
544
+ } else if (SRC_DTYPE == at::ScalarType::Half) { \
545
+ FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
546
+ } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
547
+ FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
548
+ } else { \
549
+ TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
550
+ } \
551
+ } else if (KV_DTYPE == "fp8_e5m2") { \
552
+ if (SRC_DTYPE == at::ScalarType::Float) { \
553
+ FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
554
+ } else if (SRC_DTYPE == at::ScalarType::Half) { \
555
+ FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
556
+ } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
557
+ FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
558
+ } else { \
559
+ TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
560
+ } \
561
+ } else { \
562
+ TORCH_CHECK(false, "Unsupported data type of kv cache: ", KV_DTYPE); \
563
+ } \
564
+ }
565
+
566
+} // namespace fp8
567
+#endif // not USE_ROCM
568
+} // namespace vllm
csrc/quantization/fp8_e5m2_kvcache/quant_utils.cuh
deleted
-277
@@ -1,277 +0,0 @@
1
-#pragma once
2
-
3
-#include <assert.h>
4
-#include <stdint.h>
5
-#include <float.h>
6
-#include <type_traits>
7
-#include "../../attention/attention_dtypes.h"
8
-#include "../../attention/dtype_float32.cuh"
9
-#include "../../attention/dtype_float16.cuh"
10
-#include "../../attention/dtype_bfloat16.cuh"
11
-
12
-
13
-namespace vllm {
14
-#ifdef ENABLE_FP8_E5M2
15
-namespace fp8_e5m2_unscaled {
16
-
17
-template<typename Tout, typename Tin>
18
-__inline__ __device__ Tout vec_conversion(const Tin& x)
19
-{
20
- return x;
21
-}
22
-
23
-// fp8 -> half
24
-template<>
25
-__inline__ __device__ uint16_t vec_conversion<uint16_t, uint8_t>(const uint8_t& a)
26
-{
27
- __half_raw res = __nv_cvt_fp8_to_halfraw(a, __NV_E5M2);
28
- return res.x;
29
-}
30
-
31
-// fp8x2 -> half2
32
-template<>
33
-__inline__ __device__ uint32_t vec_conversion<uint32_t, uint16_t>(const uint16_t& a)
34
-{
35
- union {
36
- uint16_t u16[2];
37
- uint32_t u32;
38
- } tmp;
39
- __half2_raw res = __nv_cvt_fp8x2_to_halfraw2(a, __NV_E5M2);
40
- tmp.u16[0] = res.x;
41
- tmp.u16[1] = res.y;
42
- return tmp.u32;
43
-}
44
-
45
-// fp8x4 -> half2x2
46
-template<>
47
-__inline__ __device__ uint2 vec_conversion<uint2, uint32_t>(const uint32_t& a)
48
-{
49
- union {
50
- uint2 u32x2;
51
- uint32_t u32[2];
52
- } tmp;
53
- tmp.u32[0] = vec_conversion<uint32_t, uint16_t>((uint16_t)a);
54
- tmp.u32[1] = vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U));
55
- return tmp.u32x2;
56
-}
57
-
58
-// fp8x8 -> half2x4
59
-template<>
60
-__inline__ __device__ uint4 vec_conversion<uint4, uint2>(const uint2& a)
61
-{
62
- union {
63
- uint4 u64x2;
64
- uint2 u64[2];
65
- } tmp;
66
- tmp.u64[0] = vec_conversion<uint2, uint32_t>(a.x);
67
- tmp.u64[1] = vec_conversion<uint2, uint32_t>(a.y);
68
- return tmp.u64x2;
69
-}
70
-
71
-// fp8 -> __nv_bfloat16
72
-template<>
73
-__inline__ __device__ __nv_bfloat16 vec_conversion<__nv_bfloat16, uint8_t>(const uint8_t& a)
74
-{
75
- // Note there is no direct convert function from fp8 to bf16.
76
- // fp8 -> half
77
- __half_raw res = __nv_cvt_fp8_to_halfraw(a, __NV_E5M2);
78
- // half -> float -> bf16
79
- float tmp = half_to_float(res.x);
80
- return __float2bfloat16(tmp);
81
-}
82
-
83
-// fp8x2 -> __nv_bfloat162
84
-template<>
85
-__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, uint16_t>(const uint16_t& a)
86
-{
87
- __nv_bfloat162 res;
88
- res.x = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a);
89
- res.y = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U));
90
- return res;
91
-}
92
-
93
-// fp8x4 -> bf16_4_t
94
-template<>
95
-__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, uint32_t>(const uint32_t& a)
96
-{
97
- bf16_4_t res;
98
- res.x = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a);
99
- res.y = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U));
100
- return res;
101
-}
102
-
103
-// fp8x8 -> bf16_8_t
104
-template<>
105
-__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, uint2>(const uint2& a)
106
-{
107
- bf16_4_t tmp1, tmp2;
108
- tmp1 = vec_conversion<bf16_4_t, uint32_t>(a.x);
109
- tmp2 = vec_conversion<bf16_4_t, uint32_t>(a.y);
110
- bf16_8_t res;
111
- res.x = tmp1.x;
112
- res.y = tmp1.y;
113
- res.z = tmp2.x;
114
- res.w = tmp2.y;
115
- return res;
116
-}
117
-
118
-// fp8 -> float
119
-template<>
120
-__inline__ __device__ float vec_conversion<float, uint8_t>(const uint8_t& a)
121
-{
122
- // fp8 -> half
123
- uint16_t tmp = vec_conversion<uint16_t, uint8_t>(a);
124
- // half -> float
125
- return half_to_float(tmp);
126
-}
127
-
128
-// fp8x2 -> float2
129
-template<>
130
-__inline__ __device__ float2 vec_conversion<float2, uint16_t>(const uint16_t& a)
131
-{
132
- // fp8x2 -> half2
133
- uint32_t tmp = vec_conversion<uint32_t, uint16_t>(a);
134
- // half2 -> float2
135
- return half2_to_float2(tmp);
136
-}
137
-
138
-// fp8x4 -> float4
139
-template<>
140
-__inline__ __device__ Float4_ vec_conversion<Float4_, uint32_t>(const uint32_t& a)
141
-{
142
- Float4_ res;
143
- res.x = vec_conversion<float2, uint16_t>((uint16_t)a);
144
- res.y = vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U));
145
- return res;
146
-}
147
-
148
-// fp8x8 -> float8
149
-template<>
150
-__inline__ __device__ Float8_ vec_conversion<Float8_, uint2>(const uint2& a)
151
-{
152
- Float4_ tmp1, tmp2;
153
- tmp1 = vec_conversion<Float4_, uint32_t>(a.x);
154
- tmp2 = vec_conversion<Float4_, uint32_t>(a.y);
155
- Float8_ res;
156
- res.x = tmp1.x;
157
- res.y = tmp1.y;
158
- res.z = tmp2.x;
159
- res.w = tmp2.y;
160
- return res;
161
-}
162
-
163
-
164
-// half -> fp8
165
-template<>
166
-__inline__ __device__ uint8_t vec_conversion<uint8_t, uint16_t>(const uint16_t& a)
167
-{
168
- __half_raw tmp;
169
- tmp.x = a;
170
- __nv_fp8_storage_t res = __nv_cvt_halfraw_to_fp8(tmp, __NV_SATFINITE, __NV_E5M2);
171
- return (uint8_t)res;
172
-}
173
-
174
-// bf16 -> fp8
175
-template<>
176
-__inline__ __device__ uint8_t vec_conversion<uint8_t, __nv_bfloat16>(const __nv_bfloat16& a)
177
-{
178
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
179
- assert(false);
180
-#else
181
- __nv_fp8_storage_t res = __nv_cvt_bfloat16raw_to_fp8(__nv_bfloat16_raw(a), __NV_SATFINITE, __NV_E5M2);
182
- return (uint8_t)res;
183
-#endif
184
-}
185
-
186
-// float -> fp8
187
-template<>
188
-__inline__ __device__ uint8_t vec_conversion<uint8_t, float>(const float& a)
189
-{
190
- __nv_fp8_storage_t res = __nv_cvt_float_to_fp8(a, __NV_SATFINITE, __NV_E5M2);
191
- return (uint8_t)res;
192
-}
193
-
194
-// fp8x4 -> float4
195
-template<>
196
-__inline__ __device__ float4 vec_conversion<float4, uint32_t>(const uint32_t& a)
197
-{
198
- Float4_ tmp = vec_conversion<Float4_, uint32_t>(a);
199
- float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
200
- return res;
201
-}
202
-
203
-
204
-template<>
205
-__inline__ __device__ uint32_t vec_conversion<uint32_t, float2>(const float2& a)
206
-{
207
- union {
208
- half2 float16;
209
- uint32_t uint32;
210
- };
211
-
212
- float16 = __float22half2_rn(a);
213
- return uint32;
214
-}
215
-
216
-template<>
217
-__inline__ __device__ uint2 vec_conversion<uint2, Float4_>(const Float4_& a)
218
-{
219
- uint2 b;
220
- float2 val;
221
- val.x = a.x.x;
222
- val.y = a.x.y;
223
- b.x = vec_conversion<uint32_t, float2>(val);
224
-
225
- val.x = a.y.x;
226
- val.y = a.y.y;
227
- b.y = vec_conversion<uint32_t, float2>(val);
228
-
229
- return b;
230
-}
231
-
232
-template<>
233
-__inline__ __device__ float4 vec_conversion<float4, Float4_>(const Float4_& a)
234
-{
235
- float4 b;
236
- b.x = a.x.x;
237
- b.y = a.x.y;
238
- b.z = a.y.x;
239
- b.w = a.y.y;
240
- return b;
241
-}
242
-
243
-template<>
244
-__inline__ __device__ uint4 vec_conversion<uint4, Float8_>(const Float8_& a)
245
-{
246
- uint4 b;
247
- b.x = vec_conversion<uint32_t, float2>(a.x);
248
- b.y = vec_conversion<uint32_t, float2>(a.y);
249
- b.z = vec_conversion<uint32_t, float2>(a.z);
250
- b.w = vec_conversion<uint32_t, float2>(a.w);
251
- return b;
252
-}
253
-
254
-template<>
255
-__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, float2>(const float2 &a) {
256
- __nv_bfloat162 b;
257
- from_float(b, a);
258
- return b;
259
-}
260
-
261
-template<>
262
-__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, Float4_>(const Float4_ &a) {
263
- bf16_4_t b;
264
- from_float(b, a);
265
- return b;
266
-}
267
-
268
-template<>
269
-__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, Float8_>(const Float8_ &a) {
270
- bf16_8_t b;
271
- from_float(b, a);
272
- return b;
273
-}
274
-
275
-} // namespace fp8_e5m2_unscaled
276
-#endif // ENABLE_FP8_E5M2
277
-} // namespace vllm
tests/kernels/test_attention.py
+2
-2
@@ -236,14 +236,14 @@ def test_paged_attention(
236
dequantized_key_cache = torch.empty(size=key_cache_shape,
237
dtype=dtype,
238
device=device)
239
- ops.convert_fp8(key_cache, dequantized_key_cache)
239
+ ops.convert_fp8(dequantized_key_cache, key_cache)
240
key_cache = dequantized_key_cache
241
242
value_cache_shape = value_cache.shape
243
dequantized_value_cache = torch.empty(size=value_cache_shape,
244
dtype=dtype,
245
device=device)
246
- ops.convert_fp8(value_cache, dequantized_value_cache)
246
+ ops.convert_fp8(dequantized_value_cache, value_cache)
247
value_cache = dequantized_value_cache
248
249
ref_output = torch.empty_like(query)
tests/kernels/test_cache.py
+11
-16
@@ -5,8 +5,6 @@ import pytest
5
import torch
6
7
from vllm import _custom_ops as ops
8
-from vllm._C import cache_ops
9
-from vllm.utils import is_hip
8
9
COPYING_DIRECTION = [('cuda', 'cpu'), ('cuda', 'cuda'), ('cpu', 'cuda')]
10
DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -25,6 +23,8 @@ SEEDS = [0]
23
CUDA_DEVICES = [
24
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
25
]
26
+
27
+# We assume fp8 is always enabled for testing.
28
KV_CACHE_DTYPE = ["auto", "fp8"]
29
30
@@ -124,8 +124,6 @@ def test_reshape_and_cache(
124
device: str,
125
kv_cache_dtype: str,
126
) -> None:
127
- if not is_hip() and kv_cache_dtype == "fp8":
128
- pytest.skip() # This test is not tuned for e5m2 cuda precision
127
random.seed(seed)
128
torch.random.manual_seed(seed)
129
if torch.cuda.is_available():
@@ -149,9 +147,9 @@ def test_reshape_and_cache(
147
# Clone the KV caches.
148
if kv_cache_dtype == "fp8":
149
cloned_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
152
- ops.convert_fp8(key_cache, cloned_key_cache)
150
+ ops.convert_fp8(cloned_key_cache, key_cache)
151
cloned_value_cache = torch.empty_like(value_cache, dtype=torch.float16)
154
- ops.convert_fp8(value_cache, cloned_value_cache)
152
+ ops.convert_fp8(cloned_value_cache, value_cache)
153
else:
154
cloned_key_cache = key_cache.clone()
155
cloned_value_cache = value_cache.clone()
@@ -165,9 +163,9 @@ def test_reshape_and_cache(
163
164
if kv_cache_dtype == "fp8":
165
result_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
168
- ops.convert_fp8(key_cache, result_key_cache)
166
+ ops.convert_fp8(result_key_cache, key_cache)
167
result_value_cache = torch.empty_like(value_cache, dtype=torch.float16)
170
- ops.convert_fp8(value_cache, result_value_cache)
168
+ ops.convert_fp8(result_value_cache, value_cache)
169
170
# Run the reference implementation.
171
reshaped_key = key.reshape(num_tokens, *key_cache[0, :, :, 0, :].shape)
@@ -255,8 +253,8 @@ def test_reshape_and_cache_flash(
253
cloned_value_cache = value_cache.clone()
254
255
# Call the reshape_and_cache kernel.
258
- cache_ops.reshape_and_cache_flash(key, value, key_cache, value_cache,
259
- slot_mapping, kv_cache_dtype)
256
+ ops.reshape_and_cache_flash(key, value, key_cache, value_cache,
257
+ slot_mapping, kv_cache_dtype)
258
259
# Run the reference implementation.
260
block_indicies = torch.div(slot_mapping, block_size, rounding_mode='floor')
@@ -299,8 +297,6 @@ def test_swap_blocks(
297
) -> None:
298
if kv_cache_dtype == "fp8" and "cpu" in direction:
299
pytest.skip()
302
- if not is_hip() and kv_cache_dtype == "fp8":
303
- pytest.skip() # This test is not tuned for e5m2 cuda precision
300
random.seed(seed)
301
torch.random.manual_seed(seed)
302
if torch.cuda.is_available():
@@ -348,7 +344,6 @@ def test_swap_blocks(
344
dist_value_caches[0][dst].cpu())
345
346
351
-@pytest.mark.skipif(not is_hip(), reason="FP8 conversion test requires e4m3")
347
@pytest.mark.parametrize("num_heads", NUM_HEADS)
348
@pytest.mark.parametrize("head_size", HEAD_SIZES)
349
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
@@ -357,7 +352,7 @@ def test_swap_blocks(
352
@pytest.mark.parametrize("seed", SEEDS)
353
@pytest.mark.parametrize("device", CUDA_DEVICES)
354
@torch.inference_mode()
360
-def test_fp8_conversion(
355
+def test_fp8_e4m3_conversion(
356
num_heads: int,
357
head_size: int,
358
block_size: int,
@@ -377,9 +372,9 @@ def test_fp8_conversion(
372
cache.uniform_(low, high)
373
374
cache_fp8 = torch.empty_like(cache, dtype=torch.uint8)
380
- ops.convert_fp8(cache, cache_fp8)
375
+ ops.convert_fp8(cache_fp8, cache)
376
377
converted_cache = torch.empty_like(cache)
383
- ops.convert_fp8(cache_fp8, converted_cache)
378
+ ops.convert_fp8(converted_cache, cache_fp8)
379
380
assert torch.allclose(cache, converted_cache, atol=0.001, rtol=0.1)
vllm/_custom_ops.py
+5
-2
@@ -270,8 +270,11 @@ def swap_blocks(src: torch.Tensor, dst: torch.Tensor,
270
vllm_cache_ops.swap_blocks(src, dst, block_mapping)
271
272
273
-def convert_fp8(output: torch.Tensor, input: torch.Tensor) -> None:
274
- vllm_cache_ops.convert_fp8(output, input)
273
+def convert_fp8(output: torch.Tensor,
274
+ input: torch.Tensor,
275
+ scale: float = 1.0,
276
+ kv_dtype: str = "fp8") -> None:
277
+ vllm_cache_ops.convert_fp8(output, input, scale, kv_dtype)
278
279
280
#TODO: cuda_utils, custom_ar
vllm/utils.py
+1
-1
@@ -329,7 +329,7 @@ def _generate_random_fp8(
329
from vllm import _custom_ops as ops
330
tensor_tmp = torch.empty_like(tensor, dtype=torch.float16)
331
tensor_tmp.uniform_(low, high)
332
- ops.convert_fp8(tensor_tmp, tensor)
332
+ ops.convert_fp8(tensor, tensor_tmp)
333
del tensor_tmp
334
335