@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c833101740
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Kernel] Refactor FP8 kv-cache with NVIDIA float8_e4m3 support (#4535)

Cody Yu committed May 9, 2024 at 17:04 UTC c83310174055bb124ea2197885b652efd59b7a0f
17 files changed +843 -558
.buildkite/check-wheel-size.py
+1 -1
@@ -1,7 +1,7 @@
1 import os
2 import zipfile
3
4 -MAX_SIZE_MB = 100
4 +MAX_SIZE_MB = 150
5
6
7 def print_top_10_largest_files(zip_file):
CMakeLists.txt
+1 -1
@@ -167,7 +167,7 @@ set(VLLM_EXT_SRC
167 "csrc/layernorm_kernels.cu"
168 "csrc/quantization/squeezellm/quant_cuda_kernel.cu"
169 "csrc/quantization/gptq/q_gemm.cu"
170 - "csrc/quantization/fp8/fp8_cuda_kernels.cu"
170 + "csrc/quantization/fp8/common.cu"
171 "csrc/cuda_utils_kernels.cu"
172 "csrc/moe_align_block_size_kernels.cu"
173 "csrc/pybind.cpp")
cmake/utils.cmake
+2 -2
@@ -99,7 +99,7 @@ function (get_torch_gpu_compiler_flags OUT_GPU_FLAGS GPU_LANG)
99 "Failed to determine torch nvcc compiler flags")
100
101 if (CUDA_VERSION VERSION_GREATER_EQUAL 11.8)
102 - list(APPEND GPU_FLAGS "-DENABLE_FP8_E5M2")
102 + list(APPEND GPU_FLAGS "-DENABLE_FP8")
103 endif()
104 if (CUDA_VERSION VERSION_GREATER_EQUAL 12.0)
105 list(REMOVE_ITEM GPU_FLAGS
@@ -119,7 +119,7 @@ function (get_torch_gpu_compiler_flags OUT_GPU_FLAGS GPU_LANG)
119
120 list(APPEND GPU_FLAGS
121 "-DUSE_ROCM"
122 - "-DENABLE_FP8_E4M3"
122 + "-DENABLE_FP8"
123 "-U__HIP_NO_HALF_CONVERSIONS__"
124 "-U__HIP_NO_HALF_OPERATORS__"
125 "-fno-gpu-rdc")
csrc/attention/attention_kernels.cu
+110 -176
@@ -19,21 +19,17 @@
19 #include <torch/extension.h>
20 #include <ATen/cuda/CUDAContext.h>
21 #include <c10/cuda/CUDAGuard.h>
22 +#include <algorithm>
23
24 #include "attention_dtypes.h"
25 #include "attention_utils.cuh"
26
26 -#if defined(ENABLE_FP8_E5M2)
27 -#include "../quantization/fp8_e5m2_kvcache/quant_utils.cuh"
28 -#elif defined(ENABLE_FP8_E4M3)
29 -#include "../quantization/fp8/amd_detail/quant_utils.cuh"
30 -#endif
31 -
32 -#include <algorithm>
33 -
27 #ifdef USE_ROCM
28 #include <hip/hip_bf16.h>
29 + #include "../quantization/fp8/amd/quant_utils.cuh"
30 typedef __hip_bfloat16 __nv_bfloat16;
31 +#else
32 + #include "../quantization/fp8/nvidia/quant_utils.cuh"
33 #endif
34
35 #ifndef USE_ROCM
@@ -92,7 +88,7 @@ template<
88 int HEAD_SIZE,
89 int BLOCK_SIZE,
90 int NUM_THREADS,
95 - bool IS_FP8_KV_CACHE,
91 + vllm::Fp8KVCacheDataType KV_DTYPE,
92 int PARTITION_SIZE = 0> // Zero means no partitioning.
93 __device__ void paged_attention_kernel(
94 float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
@@ -157,9 +153,7 @@ __device__ void paged_attention_kernel(
153 constexpr int VEC_SIZE = MAX(16 / (THREAD_GROUP_SIZE * sizeof(scalar_t)), 1);
154 using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
155 using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
160 -#if defined(ENABLE_FP8_E5M2) || defined(ENABLE_FP8_E4M3)
156 using Quant_vec = typename Vec<cache_t, VEC_SIZE>::Type;
162 -#endif
157
158 constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
159 constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
@@ -223,21 +217,14 @@ __device__ void paged_attention_kernel(
217 const int vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;
218 const int offset1 = (vec_idx * VEC_SIZE) / x;
219 const int offset2 = (vec_idx * VEC_SIZE) % x;
226 - if constexpr (IS_FP8_KV_CACHE) {
227 -#if defined(ENABLE_FP8_E5M2)
228 - Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
229 - // Vector conversion from Quant_vec to K_vec.
230 - k_vecs[j] = fp8_e5m2_unscaled::vec_conversion<K_vec, Quant_vec>(k_vec_quant);
231 -#elif defined(ENABLE_FP8_E4M3)
232 - Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
233 - // Vector conversion from Quant_vec to K_vec. Use scaled_vec_conversion to convert FP8_E4M3 quantized k
234 - // cache vec to k vec in higher precision (FP16, BFloat16, etc.)
235 - k_vecs[j] = fp8_e4m3::scaled_vec_conversion<K_vec, Quant_vec>(k_vec_quant, kv_scale);
236 -#else
237 - assert(false);
238 -#endif
239 - } else {
220 +
221 + if constexpr (KV_DTYPE == Fp8KVCacheDataType::kAuto) {
222 k_vecs[j] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
223 + } else {
224 + // Vector conversion from Quant_vec to K_vec.
225 + Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(
226 + k_ptr + offset1 * BLOCK_SIZE * x + offset2);
227 + k_vecs[j] = fp8::scaled_convert<K_vec, Quant_vec, KV_DTYPE>(k_vec_quant, kv_scale);
228 }
229 }
230
@@ -312,9 +299,7 @@ __device__ void paged_attention_kernel(
299 constexpr int V_VEC_SIZE = MIN(16 / sizeof(scalar_t), BLOCK_SIZE);
300 using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
301 using L_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
315 -#if defined(ENABLE_FP8_E5M2) || defined(ENABLE_FP8_E4M3)
302 using V_quant_vec = typename Vec<cache_t, V_VEC_SIZE>::Type;
317 -#endif
303 using Float_L_vec = typename FloatVec<L_vec>::Type;
304
305 constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
@@ -348,21 +333,13 @@ __device__ void paged_attention_kernel(
333 if (row_idx < HEAD_SIZE) {
334 const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
335 V_vec v_vec;
351 - if constexpr (IS_FP8_KV_CACHE) {
352 -#if defined(ENABLE_FP8_E5M2)
336 +
337 + if constexpr (KV_DTYPE == Fp8KVCacheDataType::kAuto) {
338 + v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
339 + } else {
340 V_quant_vec v_quant_vec = *reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
341 // Vector conversion from V_quant_vec to V_vec.
355 - v_vec = fp8_e5m2_unscaled::vec_conversion<V_vec, V_quant_vec>(v_quant_vec);
356 -#elif defined(ENABLE_FP8_E4M3)
357 - V_quant_vec v_quant_vec = *reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
358 - // Vector conversion from V_quant_vec to V_vec. Use scaled_vec_conversion to convert
359 - // FP8_E4M3 quantized v cache vec to v vec in higher precision (FP16, BFloat16, etc.)
360 - v_vec = fp8_e4m3::scaled_vec_conversion<V_vec, V_quant_vec>(v_quant_vec, kv_scale);
361 -#else
362 - assert(false);
363 -#endif
364 - } else {
365 - v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
342 + v_vec = fp8::scaled_convert<V_vec, V_quant_vec, KV_DTYPE>(v_quant_vec, kv_scale);
343 }
344 if (block_idx == num_seq_blocks - 1) {
345 // NOTE(woosuk): When v_vec contains the tokens that are out of the context,
@@ -448,7 +425,7 @@ template<
425 int HEAD_SIZE,
426 int BLOCK_SIZE,
427 int NUM_THREADS,
451 - bool IS_FP8_KV_CACHE>
428 + vllm::Fp8KVCacheDataType KV_DTYPE>
429 __global__ void paged_attention_v1_kernel(
430 scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
431 const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
@@ -464,7 +441,7 @@ __global__ void paged_attention_v1_kernel(
441 const int kv_block_stride,
442 const int kv_head_stride,
443 const float kv_scale) {
467 - paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_KV_CACHE>(
444 + paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, KV_DTYPE>(
445 /* exp_sums */ nullptr, /* max_logits */ nullptr,
446 out, q, k_cache, v_cache, num_kv_heads, scale, block_tables, seq_lens,
447 max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride, kv_head_stride, kv_scale);
@@ -477,7 +454,7 @@ template<
454 int HEAD_SIZE,
455 int BLOCK_SIZE,
456 int NUM_THREADS,
480 - bool IS_FP8_KV_CACHE,
457 + vllm::Fp8KVCacheDataType KV_DTYPE,
458 int PARTITION_SIZE>
459 __global__ void paged_attention_v2_kernel(
460 float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
@@ -496,7 +473,7 @@ __global__ void paged_attention_v2_kernel(
473 const int kv_block_stride,
474 const int kv_head_stride,
475 const float kv_scale) {
499 - paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_KV_CACHE, PARTITION_SIZE>(
476 + paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, KV_DTYPE, PARTITION_SIZE>(
477 exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
478 block_tables, seq_lens, max_num_blocks_per_seq, alibi_slopes,
479 q_stride, kv_block_stride, kv_head_stride, kv_scale);
@@ -606,9 +583,9 @@ __global__ void paged_attention_v2_reduce_kernel(
583 #define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
584 VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
585 ((void*)vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
609 - IS_FP8_KV_CACHE>), shared_mem_size); \
586 + KV_DTYPE>), shared_mem_size); \
587 vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
611 - IS_FP8_KV_CACHE><<<grid, block, shared_mem_size, stream>>>( \
588 + KV_DTYPE><<<grid, block, shared_mem_size, stream>>>( \
589 out_ptr, \
590 query_ptr, \
591 key_cache_ptr, \
@@ -629,7 +606,7 @@ template<
606 typename T,
607 typename CACHE_T,
608 int BLOCK_SIZE,
632 - bool IS_FP8_KV_CACHE,
609 + vllm::Fp8KVCacheDataType KV_DTYPE,
610 int NUM_THREADS = 128>
611 void paged_attention_v1_launcher(
612 torch::Tensor& out,
@@ -706,36 +683,36 @@ void paged_attention_v1_launcher(
683 }
684 }
685
709 -#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
710 - paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE>( \
711 - out, \
712 - query, \
713 - key_cache, \
714 - value_cache, \
715 - num_kv_heads, \
716 - scale, \
717 - block_tables, \
718 - seq_lens, \
719 - max_seq_len, \
720 - alibi_slopes, \
686 +#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE) \
687 + paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE>( \
688 + out, \
689 + query, \
690 + key_cache, \
691 + value_cache, \
692 + num_kv_heads, \
693 + scale, \
694 + block_tables, \
695 + seq_lens, \
696 + max_seq_len, \
697 + alibi_slopes, \
698 kv_scale);
699
700 // NOTE(woosuk): To reduce the compilation time, we omitted block sizes
701 // 1, 2, 4, 64, 128, 256.
725 -#define CALL_V1_LAUNCHER_BLOCK_SIZE(T, CACHE_T, IS_FP8_KV_CACHE) \
726 - switch (block_size) { \
727 - case 8: \
728 - CALL_V1_LAUNCHER(T, CACHE_T, 8, IS_FP8_KV_CACHE); \
729 - break; \
730 - case 16: \
731 - CALL_V1_LAUNCHER(T, CACHE_T, 16, IS_FP8_KV_CACHE); \
732 - break; \
733 - case 32: \
734 - CALL_V1_LAUNCHER(T, CACHE_T, 32, IS_FP8_KV_CACHE); \
735 - break; \
736 - default: \
737 - TORCH_CHECK(false, "Unsupported block size: ", block_size); \
738 - break; \
702 +#define CALL_V1_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
703 + switch (block_size) { \
704 + case 8: \
705 + CALL_V1_LAUNCHER(T, CACHE_T, 8, KV_DTYPE); \
706 + break; \
707 + case 16: \
708 + CALL_V1_LAUNCHER(T, CACHE_T, 16, KV_DTYPE); \
709 + break; \
710 + case 32: \
711 + CALL_V1_LAUNCHER(T, CACHE_T, 32, KV_DTYPE); \
712 + break; \
713 + default: \
714 + TORCH_CHECK(false, "Unsupported block size: ", block_size); \
715 + break; \
716 }
717
718 void paged_attention_v1(
@@ -752,65 +729,44 @@ void paged_attention_v1(
729 const c10::optional<torch::Tensor>& alibi_slopes,
730 const std::string& kv_cache_dtype,
731 float kv_scale) {
755 - if (kv_cache_dtype == "auto") {
756 - if (query.dtype() == at::ScalarType::Float) {
757 - CALL_V1_LAUNCHER_BLOCK_SIZE(float, float, false);
758 - } else if (query.dtype() == at::ScalarType::Half) {
759 - CALL_V1_LAUNCHER_BLOCK_SIZE(uint16_t, uint16_t, false);
760 - } else if (query.dtype() == at::ScalarType::BFloat16) {
761 - CALL_V1_LAUNCHER_BLOCK_SIZE(__nv_bfloat16, __nv_bfloat16, false);
762 - } else {
763 - TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
764 - }
765 - } else if (kv_cache_dtype == "fp8") {
766 - if (query.dtype() == at::ScalarType::Float) {
767 - CALL_V1_LAUNCHER_BLOCK_SIZE(float, uint8_t, true);
768 - } else if (query.dtype() == at::ScalarType::Half) {
769 - CALL_V1_LAUNCHER_BLOCK_SIZE(uint16_t, uint8_t, true);
770 - } else if (query.dtype() == at::ScalarType::BFloat16) {
771 - CALL_V1_LAUNCHER_BLOCK_SIZE(__nv_bfloat16, uint8_t, true);
772 - } else {
773 - TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
774 - }
775 - } else {
776 - TORCH_CHECK(false, "Unsupported data type of kv cache: ", kv_cache_dtype);
777 - }
732 +
733 + DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype, CALL_V1_LAUNCHER_BLOCK_SIZE)
734 }
735
780 -#define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
781 - vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
782 - IS_FP8_KV_CACHE, PARTITION_SIZE> \
783 - <<<grid, block, shared_mem_size, stream>>>( \
784 - exp_sums_ptr, \
785 - max_logits_ptr, \
786 - tmp_out_ptr, \
787 - query_ptr, \
788 - key_cache_ptr, \
789 - value_cache_ptr, \
790 - num_kv_heads, \
791 - scale, \
792 - block_tables_ptr, \
793 - seq_lens_ptr, \
794 - max_num_blocks_per_seq, \
795 - alibi_slopes_ptr, \
796 - q_stride, \
797 - kv_block_stride, \
798 - kv_head_stride, \
799 - kv_scale); \
800 - vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, PARTITION_SIZE> \
801 - <<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
802 - out_ptr, \
803 - exp_sums_ptr, \
804 - max_logits_ptr, \
805 - tmp_out_ptr, \
806 - seq_lens_ptr, \
736 +#define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
737 + vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
738 + KV_DTYPE, PARTITION_SIZE> \
739 + <<<grid, block, shared_mem_size, stream>>>( \
740 + exp_sums_ptr, \
741 + max_logits_ptr, \
742 + tmp_out_ptr, \
743 + query_ptr, \
744 + key_cache_ptr, \
745 + value_cache_ptr, \
746 + num_kv_heads, \
747 + scale, \
748 + block_tables_ptr, \
749 + seq_lens_ptr, \
750 + max_num_blocks_per_seq, \
751 + alibi_slopes_ptr, \
752 + q_stride, \
753 + kv_block_stride, \
754 + kv_head_stride, \
755 + kv_scale); \
756 + vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, PARTITION_SIZE> \
757 + <<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
758 + out_ptr, \
759 + exp_sums_ptr, \
760 + max_logits_ptr, \
761 + tmp_out_ptr, \
762 + seq_lens_ptr, \
763 max_num_partitions);
764
765 template<
766 typename T,
767 typename CACHE_T,
768 int BLOCK_SIZE,
813 - bool IS_FP8_KV_CACHE,
769 + vllm::Fp8KVCacheDataType KV_DTYPE,
770 int NUM_THREADS = 128,
771 int PARTITION_SIZE = 512>
772 void paged_attention_v2_launcher(
@@ -897,39 +853,39 @@ void paged_attention_v2_launcher(
853 }
854 }
855
900 -#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
901 - paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE>( \
902 - out, \
903 - exp_sums, \
904 - max_logits, \
905 - tmp_out, \
906 - query, \
907 - key_cache, \
908 - value_cache, \
909 - num_kv_heads, \
910 - scale, \
911 - block_tables, \
912 - seq_lens, \
913 - max_seq_len, \
914 - alibi_slopes, \
856 +#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE) \
857 + paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE>( \
858 + out, \
859 + exp_sums, \
860 + max_logits, \
861 + tmp_out, \
862 + query, \
863 + key_cache, \
864 + value_cache, \
865 + num_kv_heads, \
866 + scale, \
867 + block_tables, \
868 + seq_lens, \
869 + max_seq_len, \
870 + alibi_slopes, \
871 kv_scale);
872
873 // NOTE(woosuk): To reduce the compilation time, we omitted block sizes
874 // 1, 2, 4, 64, 128, 256.
919 -#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, IS_FP8_KV_CACHE) \
920 - switch (block_size) { \
921 - case 8: \
922 - CALL_V2_LAUNCHER(T, CACHE_T, 8, IS_FP8_KV_CACHE); \
923 - break; \
924 - case 16: \
925 - CALL_V2_LAUNCHER(T, CACHE_T, 16, IS_FP8_KV_CACHE); \
926 - break; \
927 - case 32: \
928 - CALL_V2_LAUNCHER(T, CACHE_T, 32, IS_FP8_KV_CACHE); \
929 - break; \
930 - default: \
931 - TORCH_CHECK(false, "Unsupported block size: ", block_size); \
932 - break; \
875 +#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
876 + switch (block_size) { \
877 + case 8: \
878 + CALL_V2_LAUNCHER(T, CACHE_T, 8, KV_DTYPE); \
879 + break; \
880 + case 16: \
881 + CALL_V2_LAUNCHER(T, CACHE_T, 16, KV_DTYPE); \
882 + break; \
883 + case 32: \
884 + CALL_V2_LAUNCHER(T, CACHE_T, 32, KV_DTYPE); \
885 + break; \
886 + default: \
887 + TORCH_CHECK(false, "Unsupported block size: ", block_size); \
888 + break; \
889 }
890
891 void paged_attention_v2(
@@ -949,29 +905,7 @@ void paged_attention_v2(
905 const c10::optional<torch::Tensor>& alibi_slopes,
906 const std::string& kv_cache_dtype,
907 float kv_scale) {
952 - if (kv_cache_dtype == "auto") {
953 - if (query.dtype() == at::ScalarType::Float) {
954 - CALL_V2_LAUNCHER_BLOCK_SIZE(float, float, false);
955 - } else if (query.dtype() == at::ScalarType::Half) {
956 - CALL_V2_LAUNCHER_BLOCK_SIZE(uint16_t, uint16_t, false);
957 - } else if (query.dtype() == at::ScalarType::BFloat16) {
958 - CALL_V2_LAUNCHER_BLOCK_SIZE(__nv_bfloat16, __nv_bfloat16, false);
959 - } else {
960 - TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
961 - }
962 - } else if (kv_cache_dtype == "fp8") {
963 - if (query.dtype() == at::ScalarType::Float) {
964 - CALL_V2_LAUNCHER_BLOCK_SIZE(float, uint8_t, true);
965 - } else if (query.dtype() == at::ScalarType::Half) {
966 - CALL_V2_LAUNCHER_BLOCK_SIZE(uint16_t, uint8_t, true);
967 - } else if (query.dtype() == at::ScalarType::BFloat16) {
968 - CALL_V2_LAUNCHER_BLOCK_SIZE(__nv_bfloat16, uint8_t, true);
969 - } else {
970 - TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
971 - }
972 - } else {
973 - TORCH_CHECK(false, "Unsupported data type of kv cache: ", kv_cache_dtype);
974 - }
908 + DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype, CALL_V2_LAUNCHER_BLOCK_SIZE)
909 }
910
911 #undef WARP_SIZE
csrc/attention/dtype_fp8.cuh
+11 -5
@@ -3,14 +3,21 @@
3 #include "attention_generic.cuh"
4
5 #include <stdint.h>
6 -#ifdef ENABLE_FP8_E5M2
6 +#ifdef ENABLE_FP8
7 +#ifndef USE_ROCM
8 #include <cuda_fp8.h>
8 -#endif
9 +#endif // USE_ROCM
10 +#endif // ENABLE_FP8
11
12 namespace vllm {
11 -#if defined(ENABLE_FP8_E5M2) || defined(ENABLE_FP8_E4M3)
12 -// fp8 vector types for quantization of kv cache
13
14 +enum class Fp8KVCacheDataType {
15 + kAuto = 0,
16 + kFp8E4M3 = 1,
17 + kFp8E5M2 = 2,
18 +};
19 +
20 +// fp8 vector types for quantization of kv cache
21 template<>
22 struct Vec<uint8_t, 1> {
23 using Type = uint8_t;
@@ -30,6 +37,5 @@ template<>
37 struct Vec<uint8_t, 8> {
38 using Type = uint2;
39 };
33 -#endif // ENABLE_FP8_E5M2
40
41 } // namespace vllm
csrc/cache.h
+3 -1
@@ -34,5 +34,7 @@ void reshape_and_cache_flash(
34
35 // Just for unittest
36 void convert_fp8(
37 + torch::Tensor& dst_cache,
38 torch::Tensor& src_cache,
38 - torch::Tensor& dst_cache);
39 + const float scale,
40 + const std::string& kv_cache_dtype);
csrc/cache_kernels.cu
+70 -73
@@ -4,10 +4,11 @@
4
5 #include "cuda_compat.h"
6 #include "dispatch_utils.h"
7 -#if defined(ENABLE_FP8_E5M2)
8 -#include "quantization/fp8_e5m2_kvcache/quant_utils.cuh"
9 -#elif defined(ENABLE_FP8_E4M3)
10 -#include "quantization/fp8/amd_detail/quant_utils.cuh"
7 +
8 +#ifdef USE_ROCM
9 +#include "quantization/fp8/amd/quant_utils.cuh"
10 +#else
11 +#include "quantization/fp8/nvidia/quant_utils.cuh"
12 #endif
13
14 #include <algorithm>
@@ -149,7 +150,7 @@ void copy_blocks(
150
151 namespace vllm {
152
152 -template<typename scalar_t, typename cache_t, bool is_fp8_kv_cache>
153 +template<typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>
154 __global__ void reshape_and_cache_kernel(
155 const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
156 const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
@@ -194,19 +195,12 @@ __global__ void reshape_and_cache_kernel(
195 + block_offset;
196 scalar_t tgt_key = key[src_key_idx];
197 scalar_t tgt_value = value[src_value_idx];
197 - if constexpr (is_fp8_kv_cache) {
198 -#if defined(ENABLE_FP8_E5M2)
199 - key_cache[tgt_key_idx] = fp8_e5m2_unscaled::vec_conversion<uint8_t, scalar_t>(tgt_key);
200 - value_cache[tgt_value_idx] = fp8_e5m2_unscaled::vec_conversion<uint8_t, scalar_t>(tgt_value);
201 -#elif defined(ENABLE_FP8_E4M3)
202 - key_cache[tgt_key_idx] = fp8_e4m3::scaled_vec_conversion<uint8_t, scalar_t>(tgt_key, kv_scale);
203 - value_cache[tgt_value_idx] = fp8_e4m3::scaled_vec_conversion<uint8_t, scalar_t>(tgt_value, kv_scale);
204 -#else
205 - assert(false);
206 -#endif
207 - } else {
198 + if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) {
199 key_cache[tgt_key_idx] = tgt_key;
200 value_cache[tgt_value_idx] = tgt_value;
201 + } else {
202 + key_cache[tgt_key_idx] = fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, kv_scale);
203 + value_cache[tgt_value_idx] = fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, kv_scale);
204 }
205 }
206 }
@@ -248,19 +242,22 @@ __global__ void reshape_and_cache_flash_kernel(
242 }
243 } // namespace vllm
244
251 -#define CALL_RESHAPE_AND_CACHE(KV_T, CACHE_T, IS_FP8_KV_CACHE) \
252 - vllm::reshape_and_cache_kernel<KV_T, CACHE_T, IS_FP8_KV_CACHE><<<grid, block, 0, stream>>>( \
253 - reinterpret_cast<KV_T*>(key.data_ptr()), \
254 - reinterpret_cast<KV_T*>(value.data_ptr()), \
255 - reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
256 - reinterpret_cast<CACHE_T*>(value_cache.data_ptr()), \
257 - slot_mapping.data_ptr<int64_t>(), \
258 - key_stride, \
259 - value_stride, \
260 - num_heads, \
261 - head_size, \
262 - block_size, \
263 - x, \
245 +// KV_T is the stored data type of kv-cache.
246 +// CACHE_T is the data type of key and value tensors.
247 +// KV_DTYPE is the real data type of kv-cache.
248 +#define CALL_RESHAPE_AND_CACHE(KV_T, CACHE_T, KV_DTYPE) \
249 + vllm::reshape_and_cache_kernel<KV_T, CACHE_T, KV_DTYPE><<<grid, block, 0, stream>>>( \
250 + reinterpret_cast<KV_T*>(key.data_ptr()), \
251 + reinterpret_cast<KV_T*>(value.data_ptr()), \
252 + reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
253 + reinterpret_cast<CACHE_T*>(value_cache.data_ptr()), \
254 + slot_mapping.data_ptr<int64_t>(), \
255 + key_stride, \
256 + value_stride, \
257 + num_heads, \
258 + head_size, \
259 + block_size, \
260 + x, \
261 kv_scale);
262
263 void reshape_and_cache(
@@ -285,25 +282,8 @@ void reshape_and_cache(
282 dim3 block(std::min(num_heads * head_size, 512));
283 const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
284 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
288 - if (kv_cache_dtype == "auto") {
289 - if (key.dtype() == at::ScalarType::Float) {
290 - CALL_RESHAPE_AND_CACHE(float, float, false);
291 - } else if (key.dtype() == at::ScalarType::Half) {
292 - CALL_RESHAPE_AND_CACHE(uint16_t, uint16_t, false);
293 - } else if (key.dtype() == at::ScalarType::BFloat16) {
294 - CALL_RESHAPE_AND_CACHE(__nv_bfloat16, __nv_bfloat16, false);
295 - }
296 - } else if (kv_cache_dtype == "fp8") {
297 - if (key.dtype() == at::ScalarType::Float) {
298 - CALL_RESHAPE_AND_CACHE(float, uint8_t, true);
299 - } else if (key.dtype() == at::ScalarType::Half) {
300 - CALL_RESHAPE_AND_CACHE(uint16_t, uint8_t, true);
301 - } else if (key.dtype() == at::ScalarType::BFloat16) {
302 - CALL_RESHAPE_AND_CACHE(__nv_bfloat16, uint8_t, true);
303 - }
304 - } else {
305 - TORCH_CHECK(false, "Unsupported data type of kv cache: ", kv_cache_dtype);
306 - }
285 +
286 + DISPATCH_BY_KV_CACHE_DTYPE(key.dtype(), kv_cache_dtype, CALL_RESHAPE_AND_CACHE)
287 }
288
289 void reshape_and_cache_flash(
@@ -353,35 +333,34 @@ void reshape_and_cache_flash(
333
334 namespace vllm {
335
356 -template<typename Tout, typename Tin>
336 +template<typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
337 __global__ void convert_fp8_kernel(
338 const Tin* __restrict__ src_cache,
339 Tout* __restrict__ dst_cache,
340 + const float kv_scale,
341 const int64_t block_stride) {
342 const int64_t block_idx = blockIdx.x;
343 for (int i = threadIdx.x; i < block_stride; i += blockDim.x) {
344 int64_t idx = block_idx * block_stride + i;
364 -#if defined(ENABLE_FP8_E5M2)
365 - dst_cache[idx] = fp8_e5m2_unscaled::vec_conversion<Tout, Tin>(src_cache[idx]);
366 -#elif defined(ENABLE_FP8_E4M3)
367 - dst_cache[idx] = fp8_e4m3::vec_conversion<Tout, Tin>(src_cache[idx]);
368 -#else
369 - assert(false);
370 -#endif
345 + dst_cache[idx] = fp8::scaled_convert<Tout, Tin, kv_dt>(src_cache[idx], kv_scale);
346 }
347 }
348
349 } // namespace vllm
350
376 -#define CALL_CONVERT_FP8(Tout, Tin) \
377 - vllm::convert_fp8_kernel<Tout, Tin><<<grid, block, 0, stream>>>( \
378 - reinterpret_cast<Tin*>(src_cache.data_ptr()), \
379 - reinterpret_cast<Tout*>(dst_cache.data_ptr()), \
351 +#define CALL_CONVERT_FP8(Tout, Tin, KV_DTYPE) \
352 + vllm::convert_fp8_kernel<Tout, Tin, KV_DTYPE><<<grid, block, 0, stream>>>( \
353 + reinterpret_cast<Tin*>(src_cache.data_ptr()), \
354 + reinterpret_cast<Tout*>(dst_cache.data_ptr()), \
355 + kv_scale, \
356 block_stride);
357
358 +// Only for testing.
359 void convert_fp8(
360 + torch::Tensor& dst_cache,
361 torch::Tensor& src_cache,
384 - torch::Tensor& dst_cache)
362 + const float kv_scale,
363 + const std::string& kv_cache_dtype)
364 {
365 torch::Device src_device = src_cache.device();
366 torch::Device dst_device = dst_cache.device();
@@ -399,17 +378,35 @@ void convert_fp8(
378 dim3 block(std::min(block_stride, int64_t(512)));
379 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
380
402 - if (src_cache.dtype() == at::ScalarType::Float) {
403 - CALL_CONVERT_FP8(uint8_t, float);
404 - } else if (src_cache.dtype() == at::ScalarType::Half) {
405 - CALL_CONVERT_FP8(uint8_t, uint16_t);
406 - } else if (src_cache.dtype() == at::ScalarType::BFloat16) {
407 - CALL_CONVERT_FP8(uint8_t, __nv_bfloat16);
408 - } else if (dst_cache.dtype() == at::ScalarType::Float) {
409 - CALL_CONVERT_FP8(float, uint8_t);
410 - } else if (dst_cache.dtype() == at::ScalarType::Half) {
411 - CALL_CONVERT_FP8(uint16_t, uint8_t);
412 - } else if (dst_cache.dtype() == at::ScalarType::BFloat16) {
413 - CALL_CONVERT_FP8(__nv_bfloat16, uint8_t);
381 + if (kv_cache_dtype == "auto") {
382 + if (src_cache.dtype() == at::ScalarType::Float) {
383 + CALL_CONVERT_FP8(uint8_t, float, vllm::Fp8KVCacheDataType::kAuto);
384 + } else if (src_cache.dtype() == at::ScalarType::Half) {
385 + CALL_CONVERT_FP8(uint8_t, uint16_t, vllm::Fp8KVCacheDataType::kAuto);
386 + } else if (src_cache.dtype() == at::ScalarType::BFloat16) {
387 + CALL_CONVERT_FP8(uint8_t, __nv_bfloat16, vllm::Fp8KVCacheDataType::kAuto);
388 + } else if (dst_cache.dtype() == at::ScalarType::Float) {
389 + CALL_CONVERT_FP8(float, uint8_t, vllm::Fp8KVCacheDataType::kAuto);
390 + } else if (dst_cache.dtype() == at::ScalarType::Half) {
391 + CALL_CONVERT_FP8(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kAuto);
392 + } else if (dst_cache.dtype() == at::ScalarType::BFloat16) {
393 + CALL_CONVERT_FP8(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kAuto);
394 + }
395 + } else if (kv_cache_dtype == "fp8" || kv_cache_dtype == "fp8_e4m3") {
396 + if (src_cache.dtype() == at::ScalarType::Float) {
397 + CALL_CONVERT_FP8(uint8_t, float, vllm::Fp8KVCacheDataType::kFp8E4M3);
398 + } else if (src_cache.dtype() == at::ScalarType::Half) {
399 + CALL_CONVERT_FP8(uint8_t, uint16_t, vllm::Fp8KVCacheDataType::kFp8E4M3);
400 + } else if (src_cache.dtype() == at::ScalarType::BFloat16) {
401 + CALL_CONVERT_FP8(uint8_t, __nv_bfloat16, vllm::Fp8KVCacheDataType::kFp8E4M3);
402 + } else if (dst_cache.dtype() == at::ScalarType::Float) {
403 + CALL_CONVERT_FP8(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3);
404 + } else if (dst_cache.dtype() == at::ScalarType::Half) {
405 + CALL_CONVERT_FP8(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3);
406 + } else if (dst_cache.dtype() == at::ScalarType::BFloat16) {
407 + CALL_CONVERT_FP8(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3);
408 + }
409 + } else {
410 + TORCH_CHECK(false, "Unsupported data type: ", kv_cache_dtype);
411 }
412 }
csrc/quantization/fp8/amd/hip_float8.h renamed
csrc/quantization/fp8/amd/hip_float8_impl.h renamed
csrc/quantization/fp8/amd/quant_utils.cuh renamed
+58 -1
@@ -5,12 +5,17 @@
5 #include <hip/hip_bf16.h>
6 #include <hip/hip_bfloat16.h>
7
8 +#include "../../../attention/dtype_fp8.cuh"
9 #include "../../../attention/dtype_float32.cuh"
10 #include "../../../attention/dtype_bfloat16.cuh"
11
12 namespace vllm
13 {
13 -namespace fp8_e4m3 {
14 +#ifdef USE_ROCM
15 +
16 +namespace fp8 {
17 +#ifdef ENABLE_FP8
18 +
19 template <typename Tout, typename Tin>
20 __inline__ __device__ Tout vec_conversion(const Tin& x)
21 {
@@ -512,6 +517,58 @@ __inline__ __device__ float4 scaled_vec_conversion<float4, uint32_t>(const uint3
517 float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
518 return res;
519 }
520 +#endif // ENABLE_FP8
521
522 +template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
523 +__inline__ __device__ Tout convert(const Tin &x) {
524 +#ifdef ENABLE_FP8
525 + if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E4M3) {
526 + return vec_conversion<Tout, Tin>(x);
527 + }
528 +#endif
529 + assert(false);
530 }
531 +
532 +template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
533 +__inline__ __device__ Tout scaled_convert(const Tin &x, const float scale) {
534 +#ifdef ENABLE_FP8
535 + if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E4M3) {
536 + return scaled_vec_conversion<Tout, Tin>(x, scale);
537 + }
538 +#endif
539 + assert(false);
540 +}
541 +
542 +// The following macro is used to dispatch the conversion function based on the
543 +// data type of the key and value cache. The FN is a macro that calls a function
544 +// with template<typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>.
545 +#define DISPATCH_BY_KV_CACHE_DTYPE(SRC_DTYPE, KV_DTYPE, FN) \
546 + if (KV_DTYPE == "auto") { \
547 + if (SRC_DTYPE == at::ScalarType::Float) { \
548 + FN(float, float, vllm::Fp8KVCacheDataType::kAuto); \
549 + } else if (SRC_DTYPE == at::ScalarType::Half) { \
550 + FN(uint16_t, uint16_t, vllm::Fp8KVCacheDataType::kAuto); \
551 + } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
552 + FN(__nv_bfloat16, __nv_bfloat16, vllm::Fp8KVCacheDataType::kAuto); \
553 + } else { \
554 + TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
555 + } \
556 + } else { \
557 + if (KV_DTYPE == "fp8" || KV_DTYPE == "fp8_e4m3") { \
558 + if (SRC_DTYPE == at::ScalarType::Float) { \
559 + FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
560 + } else if (SRC_DTYPE == at::ScalarType::Half) { \
561 + FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
562 + } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
563 + FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
564 + } else { \
565 + TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
566 + } \
567 + } else { \
568 + TORCH_CHECK(false, "Unsupported data type of kv cache: ", KV_DTYPE); \
569 + } \
570 + }
571 +
572 +} // fp8
573 +#endif // USE_ROCM
574 } // namespace vllm
csrc/quantization/fp8/common.cu renamed
csrc/quantization/fp8/nvidia/quant_utils.cuh new
+568
@@ -0,0 +1,568 @@
1 +#pragma once
2 +
3 +#include "../../../attention/attention_dtypes.h"
4 +#include <assert.h>
5 +#include <float.h>
6 +#include <stdint.h>
7 +#include <type_traits>
8 +
9 +namespace vllm {
10 +#ifndef USE_ROCM
11 +
12 +namespace fp8 {
13 +#ifdef ENABLE_FP8
14 +
15 +#if 0 // Disable the following code to reduce the binary size.
16 +template <typename Tout, typename Tin>
17 +__inline__ __device__ Tout
18 +vec_conversion(const Tin &x, const __nv_fp8_interpretation_t fp8_type) {
19 + return x;
20 +}
21 +
22 +// fp8 -> half
23 +template <>
24 +__inline__ __device__ uint16_t vec_conversion<uint16_t, uint8_t>(
25 + const uint8_t &a, const __nv_fp8_interpretation_t fp8_type) {
26 + __half_raw res = __nv_cvt_fp8_to_halfraw(a, fp8_type);
27 + return res.x;
28 +}
29 +
30 +// fp8x2 -> half2
31 +template <>
32 +__inline__ __device__ uint32_t vec_conversion<uint32_t, uint16_t>(
33 + const uint16_t &a, const __nv_fp8_interpretation_t fp8_type) {
34 + union {
35 + uint16_t u16[2];
36 + uint32_t u32;
37 + } tmp;
38 + __half2_raw res = __nv_cvt_fp8x2_to_halfraw2(a, fp8_type);
39 + tmp.u16[0] = res.x;
40 + tmp.u16[1] = res.y;
41 + return tmp.u32;
42 +}
43 +
44 +// fp8x4 -> half2x2
45 +template <>
46 +__inline__ __device__ uint2 vec_conversion<uint2, uint32_t>(
47 + const uint32_t &a, const __nv_fp8_interpretation_t fp8_type) {
48 + union {
49 + uint2 u32x2;
50 + uint32_t u32[2];
51 + } tmp;
52 + tmp.u32[0] = vec_conversion<uint32_t, uint16_t>((uint16_t)a, fp8_type);
53 + tmp.u32[1] =
54 + vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U), fp8_type);
55 + return tmp.u32x2;
56 +}
57 +
58 +// fp8x8 -> half2x4
59 +template <>
60 +__inline__ __device__ uint4 vec_conversion<uint4, uint2>(
61 + const uint2 &a, const __nv_fp8_interpretation_t fp8_type) {
62 + union {
63 + uint4 u64x2;
64 + uint2 u64[2];
65 + } tmp;
66 + tmp.u64[0] = vec_conversion<uint2, uint32_t>(a.x, fp8_type);
67 + tmp.u64[1] = vec_conversion<uint2, uint32_t>(a.y, fp8_type);
68 + return tmp.u64x2;
69 +}
70 +
71 +// fp8 -> __nv_bfloat16
72 +template <>
73 +__inline__ __device__ __nv_bfloat16 vec_conversion<__nv_bfloat16, uint8_t>(
74 + const uint8_t &a, const __nv_fp8_interpretation_t fp8_type) {
75 + // Note there is no direct convert function from fp8 to bf16.
76 + // fp8 -> half
77 + __half_raw res = __nv_cvt_fp8_to_halfraw(a, fp8_type);
78 + // half -> float -> bf16
79 + float tmp = half_to_float(res.x);
80 + return __float2bfloat16(tmp);
81 +}
82 +
83 +// fp8x2 -> __nv_bfloat162
84 +template <>
85 +__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, uint16_t>(
86 + const uint16_t &a, const __nv_fp8_interpretation_t fp8_type) {
87 + __nv_bfloat162 res;
88 + res.x = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a, fp8_type);
89 + res.y = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U), fp8_type);
90 + return res;
91 +}
92 +
93 +// fp8x4 -> bf16_4_t
94 +template <>
95 +__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, uint32_t>(
96 + const uint32_t &a, const __nv_fp8_interpretation_t fp8_type) {
97 + bf16_4_t res;
98 + res.x = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a, fp8_type);
99 + res.y =
100 + vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U), fp8_type);
101 + return res;
102 +}
103 +
104 +// fp8x8 -> bf16_8_t
105 +template <>
106 +__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, uint2>(
107 + const uint2 &a, const __nv_fp8_interpretation_t fp8_type) {
108 + bf16_4_t tmp1, tmp2;
109 + tmp1 = vec_conversion<bf16_4_t, uint32_t>(a.x, fp8_type);
110 + tmp2 = vec_conversion<bf16_4_t, uint32_t>(a.y, fp8_type);
111 + bf16_8_t res;
112 + res.x = tmp1.x;
113 + res.y = tmp1.y;
114 + res.z = tmp2.x;
115 + res.w = tmp2.y;
116 + return res;
117 +}
118 +
119 +// fp8 -> float
120 +template <>
121 +__inline__ __device__ float
122 +vec_conversion<float, uint8_t>(const uint8_t &a,
123 + const __nv_fp8_interpretation_t fp8_type) {
124 + // fp8 -> half
125 + uint16_t tmp = vec_conversion<uint16_t, uint8_t>(a, fp8_type);
126 + // half -> float
127 + return half_to_float(tmp);
128 +}
129 +
130 +// fp8x2 -> float2
131 +template <>
132 +__inline__ __device__ float2 vec_conversion<float2, uint16_t>(
133 + const uint16_t &a, const __nv_fp8_interpretation_t fp8_type) {
134 + // fp8x2 -> half2
135 + uint32_t tmp = vec_conversion<uint32_t, uint16_t>(a, fp8_type);
136 + // half2 -> float2
137 + return half2_to_float2(tmp);
138 +}
139 +
140 +// fp8x4 -> float4
141 +template <>
142 +__inline__ __device__ Float4_ vec_conversion<Float4_, uint32_t>(
143 + const uint32_t &a, const __nv_fp8_interpretation_t fp8_type) {
144 + Float4_ res;
145 + res.x = vec_conversion<float2, uint16_t>((uint16_t)a, fp8_type);
146 + res.y = vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U), fp8_type);
147 + return res;
148 +}
149 +
150 +// fp8x8 -> float8
151 +template <>
152 +__inline__ __device__ Float8_ vec_conversion<Float8_, uint2>(
153 + const uint2 &a, const __nv_fp8_interpretation_t fp8_type) {
154 + Float4_ tmp1, tmp2;
155 + tmp1 = vec_conversion<Float4_, uint32_t>(a.x, fp8_type);
156 + tmp2 = vec_conversion<Float4_, uint32_t>(a.y, fp8_type);
157 + Float8_ res;
158 + res.x = tmp1.x;
159 + res.y = tmp1.y;
160 + res.z = tmp2.x;
161 + res.w = tmp2.y;
162 + return res;
163 +}
164 +
165 +// half -> fp8
166 +template <>
167 +__inline__ __device__ uint8_t vec_conversion<uint8_t, uint16_t>(
168 + const uint16_t &a, const __nv_fp8_interpretation_t fp8_type) {
169 + __half_raw tmp;
170 + tmp.x = a;
171 + __nv_fp8_storage_t res =
172 + __nv_cvt_halfraw_to_fp8(tmp, __NV_SATFINITE, fp8_type);
173 + return (uint8_t)res;
174 +}
175 +
176 +// bf16 -> fp8
177 +template <>
178 +__inline__ __device__ uint8_t vec_conversion<uint8_t, __nv_bfloat16>(
179 + const __nv_bfloat16 &a, const __nv_fp8_interpretation_t fp8_type) {
180 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
181 + assert(false);
182 +#else
183 + __nv_fp8_storage_t res = __nv_cvt_bfloat16raw_to_fp8(
184 + __nv_bfloat16_raw(a), __NV_SATFINITE, fp8_type);
185 + return (uint8_t)res;
186 +#endif
187 +}
188 +
189 +// float -> fp8
190 +template <>
191 +__inline__ __device__ uint8_t vec_conversion<uint8_t, float>(
192 + const float &a, const __nv_fp8_interpretation_t fp8_type) {
193 + __nv_fp8_storage_t res = __nv_cvt_float_to_fp8(a, __NV_SATFINITE, fp8_type);
194 + return (uint8_t)res;
195 +}
196 +
197 +// fp8x4 -> float4
198 +template <>
199 +__inline__ __device__ float4 vec_conversion<float4, uint32_t>(
200 + const uint32_t &a, const __nv_fp8_interpretation_t fp8_type) {
201 + Float4_ tmp = vec_conversion<Float4_, uint32_t>(a, fp8_type);
202 + float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
203 + return res;
204 +}
205 +
206 +template <>
207 +__inline__ __device__ uint32_t vec_conversion<uint32_t, float2>(
208 + const float2 &a, const __nv_fp8_interpretation_t fp8_type) {
209 + union {
210 + half2 float16;
211 + uint32_t uint32;
212 + };
213 +
214 + float16 = __float22half2_rn(a);
215 + return uint32;
216 +}
217 +
218 +template <>
219 +__inline__ __device__ uint2 vec_conversion<uint2, Float4_>(
220 + const Float4_ &a, const __nv_fp8_interpretation_t fp8_type) {
221 + uint2 b;
222 + float2 val;
223 + val.x = a.x.x;
224 + val.y = a.x.y;
225 + b.x = vec_conversion<uint32_t, float2>(val, fp8_type);
226 +
227 + val.x = a.y.x;
228 + val.y = a.y.y;
229 + b.y = vec_conversion<uint32_t, float2>(val, fp8_type);
230 +
231 + return b;
232 +}
233 +
234 +template <>
235 +__inline__ __device__ float4 vec_conversion<float4, Float4_>(
236 + const Float4_ &a, const __nv_fp8_interpretation_t fp8_type) {
237 + float4 b;
238 + b.x = a.x.x;
239 + b.y = a.x.y;
240 + b.z = a.y.x;
241 + b.w = a.y.y;
242 + return b;
243 +}
244 +
245 +template <>
246 +__inline__ __device__ uint4 vec_conversion<uint4, Float8_>(
247 + const Float8_ &a, const __nv_fp8_interpretation_t fp8_type) {
248 + uint4 b;
249 + b.x = vec_conversion<uint32_t, float2>(a.x, fp8_type);
250 + b.y = vec_conversion<uint32_t, float2>(a.y, fp8_type);
251 + b.z = vec_conversion<uint32_t, float2>(a.z, fp8_type);
252 + b.w = vec_conversion<uint32_t, float2>(a.w, fp8_type);
253 + return b;
254 +}
255 +
256 +template <>
257 +__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, float2>(
258 + const float2 &a, const __nv_fp8_interpretation_t fp8_type) {
259 + __nv_bfloat162 b;
260 + from_float(b, a);
261 + return b;
262 +}
263 +
264 +template <>
265 +__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, Float4_>(
266 + const Float4_ &a, const __nv_fp8_interpretation_t fp8_type) {
267 + bf16_4_t b;
268 + from_float(b, a);
269 + return b;
270 +}
271 +
272 +template <>
273 +__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, Float8_>(
274 + const Float8_ &a, const __nv_fp8_interpretation_t fp8_type) {
275 + bf16_8_t b;
276 + from_float(b, a);
277 + return b;
278 +}
279 +#endif
280 +
281 +/* Scaled and vectorized conversions, for data exchange between high and low
282 + precision domains Convention of the scale in API, e.g: FP8_data =
283 + Quantization( High_Precision_data / scale ) s.t. Quantize(HP / scale) => FP8
284 + Dequant(FP8) * scale => HP
285 + */
286 +
287 +template <typename Tout, typename Tin>
288 +__inline__ __device__ Tout scaled_vec_conversion(
289 + const Tin &x, const float scale, const __nv_fp8_interpretation_t fp8_type) {
290 + return x;
291 +}
292 +
293 +// fp8 -> half
294 +template <>
295 +__inline__ __device__ uint16_t scaled_vec_conversion<uint16_t, uint8_t>(
296 + const uint8_t &a, const float scale,
297 + const __nv_fp8_interpretation_t fp8_type) {
298 + __half_raw tmp = __nv_cvt_fp8_to_halfraw(a, fp8_type);
299 + return float_to_half(half_to_float(tmp.x) * scale);
300 +}
301 +
302 +// fp8x2 -> half2
303 +template <>
304 +__inline__ __device__ uint32_t scaled_vec_conversion<uint32_t, uint16_t>(
305 + const uint16_t &a, const float scale,
306 + const __nv_fp8_interpretation_t fp8_type) {
307 + union {
308 + uint16_t u16[2];
309 + uint32_t u32;
310 + } tmp;
311 + __half2_raw res = __nv_cvt_fp8x2_to_halfraw2(a, fp8_type);
312 + tmp.u16[0] = float_to_half(half_to_float(res.x) * scale);
313 + tmp.u16[1] = float_to_half(half_to_float(res.y) * scale);
314 + return tmp.u32;
315 +}
316 +
317 +// fp8x4 -> half2x2
318 +template <>
319 +__inline__ __device__ uint2 scaled_vec_conversion<uint2, uint32_t>(
320 + const uint32_t &a, const float scale,
321 + const __nv_fp8_interpretation_t fp8_type) {
322 + union {
323 + uint2 u32x2;
324 + uint32_t u32[2];
325 + } tmp;
326 + tmp.u32[0] =
327 + scaled_vec_conversion<uint32_t, uint16_t>((uint16_t)a, scale, fp8_type);
328 + tmp.u32[1] = scaled_vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U),
329 + scale, fp8_type);
330 + return tmp.u32x2;
331 +}
332 +
333 +// fp8x8 -> half2x4
334 +template <>
335 +__inline__ __device__ uint4
336 +scaled_vec_conversion<uint4, uint2>(const uint2 &a, const float scale,
337 + const __nv_fp8_interpretation_t fp8_type) {
338 + union {
339 + uint4 u64x2;
340 + uint2 u64[2];
341 + } tmp;
342 + tmp.u64[0] = scaled_vec_conversion<uint2, uint32_t>(a.x, scale, fp8_type);
343 + tmp.u64[1] = scaled_vec_conversion<uint2, uint32_t>(a.y, scale, fp8_type);
344 + return tmp.u64x2;
345 +}
346 +
347 +// fp8 -> __nv_bfloat16
348 +template <>
349 +__inline__ __device__ __nv_bfloat16
350 +scaled_vec_conversion<__nv_bfloat16, uint8_t>(
351 + const uint8_t &a, const float scale,
352 + const __nv_fp8_interpretation_t fp8_type) {
353 + // Note there is no direct convert function from fp8 to bf16.
354 + // fp8 -> half
355 + __half_raw res = __nv_cvt_fp8_to_halfraw(a, fp8_type);
356 + // half -> float -> bf16
357 + float tmp = half_to_float(res.x);
358 + return __float2bfloat16(tmp * scale);
359 +}
360 +
361 +// fp8x2 -> __nv_bfloat162
362 +template <>
363 +__inline__ __device__ __nv_bfloat162
364 +scaled_vec_conversion<__nv_bfloat162, uint16_t>(
365 + const uint16_t &a, const float scale,
366 + const __nv_fp8_interpretation_t fp8_type) {
367 + __nv_bfloat162 res;
368 + res.x = scaled_vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a, scale,
369 + fp8_type);
370 + res.y = scaled_vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U),
371 + scale, fp8_type);
372 + return res;
373 +}
374 +
375 +// fp8x4 -> bf16_4_t
376 +template <>
377 +__inline__ __device__ bf16_4_t scaled_vec_conversion<bf16_4_t, uint32_t>(
378 + const uint32_t &a, const float scale,
379 + const __nv_fp8_interpretation_t fp8_type) {
380 + bf16_4_t res;
381 + res.x = scaled_vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a, scale,
382 + fp8_type);
383 + res.y = scaled_vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U),
384 + scale, fp8_type);
385 + return res;
386 +}
387 +
388 +// fp8x8 -> bf16_8_t
389 +template <>
390 +__inline__ __device__ bf16_8_t scaled_vec_conversion<bf16_8_t, uint2>(
391 + const uint2 &a, const float scale,
392 + const __nv_fp8_interpretation_t fp8_type) {
393 + bf16_4_t tmp1, tmp2;
394 + tmp1 = scaled_vec_conversion<bf16_4_t, uint32_t>(a.x, scale, fp8_type);
395 + tmp2 = scaled_vec_conversion<bf16_4_t, uint32_t>(a.y, scale, fp8_type);
396 + bf16_8_t res;
397 + res.x = tmp1.x;
398 + res.y = tmp1.y;
399 + res.z = tmp2.x;
400 + res.w = tmp2.y;
401 + return res;
402 +}
403 +
404 +// fp8 -> float
405 +template <>
406 +__inline__ __device__ float scaled_vec_conversion<float, uint8_t>(
407 + const uint8_t &a, const float scale,
408 + const __nv_fp8_interpretation_t fp8_type) {
409 +
410 + // fp8 -> half
411 + __half_raw res = __nv_cvt_fp8_to_halfraw(a, fp8_type);
412 + uint16_t tmp = res.x;
413 +
414 + // half -> float
415 + return half_to_float(tmp) * scale;
416 +}
417 +
418 +// fp8x2 -> float2
419 +template <>
420 +__inline__ __device__ float2 scaled_vec_conversion<float2, uint16_t>(
421 + const uint16_t &a, const float scale,
422 + const __nv_fp8_interpretation_t fp8_type) {
423 + // fp8x2 -> half2
424 + uint32_t tmp = scaled_vec_conversion<uint32_t, uint16_t>(a, scale, fp8_type);
425 + // half2 -> float2
426 + return half2_to_float2(tmp);
427 +}
428 +
429 +// fp8x4 -> float4
430 +template <>
431 +__inline__ __device__ Float4_ scaled_vec_conversion<Float4_, uint32_t>(
432 + const uint32_t &a, const float scale,
433 + const __nv_fp8_interpretation_t fp8_type) {
434 + Float4_ res;
435 + res.x = scaled_vec_conversion<float2, uint16_t>((uint16_t)a, scale, fp8_type);
436 + res.y = scaled_vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U), scale,
437 + fp8_type);
438 + return res;
439 +}
440 +
441 +// fp8x8 -> float8
442 +template <>
443 +__inline__ __device__ Float8_ scaled_vec_conversion<Float8_, uint2>(
444 + const uint2 &a, const float scale,
445 + const __nv_fp8_interpretation_t fp8_type) {
446 + Float4_ tmp1, tmp2;
447 + tmp1 = scaled_vec_conversion<Float4_, uint32_t>(a.x, scale, fp8_type);
448 + tmp2 = scaled_vec_conversion<Float4_, uint32_t>(a.y, scale, fp8_type);
449 + Float8_ res;
450 + res.x = tmp1.x;
451 + res.y = tmp1.y;
452 + res.z = tmp2.x;
453 + res.w = tmp2.y;
454 + return res;
455 +}
456 +
457 +// half -> fp8
458 +template <>
459 +__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, uint16_t>(
460 + const uint16_t &a, const float scale,
461 + const __nv_fp8_interpretation_t fp8_type) {
462 + __nv_fp8_storage_t res =
463 + __nv_cvt_float_to_fp8(half_to_float(a) / scale, __NV_SATFINITE, fp8_type);
464 + return (uint8_t)res;
465 +}
466 +
467 +// bf16 -> fp8
468 +template <>
469 +__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, __nv_bfloat16>(
470 + const __nv_bfloat16 &a, const float scale,
471 + const __nv_fp8_interpretation_t fp8_type) {
472 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
473 + assert(false);
474 +#else
475 + __nv_fp8_storage_t res = __nv_cvt_float_to_fp8(__bfloat162float(a) / scale,
476 + __NV_SATFINITE, fp8_type);
477 + return (uint8_t)res;
478 +#endif
479 +}
480 +
481 +// float -> fp8
482 +template <>
483 +__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, float>(
484 + const float &a, const float scale,
485 + const __nv_fp8_interpretation_t fp8_type) {
486 + __nv_fp8_storage_t res =
487 + __nv_cvt_float_to_fp8(a / scale, __NV_SATFINITE, fp8_type);
488 + return (uint8_t)res;
489 +}
490 +
491 +// fp8x4 -> float4
492 +template <>
493 +__inline__ __device__ float4 scaled_vec_conversion<float4, uint32_t>(
494 + const uint32_t &a, const float scale,
495 + const __nv_fp8_interpretation_t fp8_type) {
496 + Float4_ tmp = scaled_vec_conversion<Float4_, uint32_t>(a, scale, fp8_type);
497 + float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
498 + return res;
499 +}
500 +#endif // ENABLE_FP8
501 +
502 +template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
503 +__inline__ __device__ Tout convert(const Tin &x) {
504 +#if 0 // Disable the following code to reduce the binary size.
505 + if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E4M3) {
506 + return vec_conversion<Tout, Tin>(x, __NV_E4M3);
507 + } else if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E5M2) {
508 + return vec_conversion<Tout, Tin>(x, __NV_E5M2);
509 + }
510 +#endif
511 + assert(false);
512 +}
513 +
514 +template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
515 +__inline__ __device__ Tout scaled_convert(const Tin &x, const float scale) {
516 +#ifdef ENABLE_FP8
517 + if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E4M3) {
518 + return scaled_vec_conversion<Tout, Tin>(x, scale, __NV_E4M3);
519 + } else if constexpr (kv_dt == Fp8KVCacheDataType::kFp8E5M2) {
520 + return scaled_vec_conversion<Tout, Tin>(x, scale, __NV_E5M2);
521 + }
522 +#endif
523 + assert(false);
524 +}
525 +
526 +// The following macro is used to dispatch the conversion function based on the
527 +// data type of the key and value cache. The FN is a macro that calls a function
528 +// with template<typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>.
529 +#define DISPATCH_BY_KV_CACHE_DTYPE(SRC_DTYPE, KV_DTYPE, FN) \
530 + if (KV_DTYPE == "auto") { \
531 + if (SRC_DTYPE == at::ScalarType::Float) { \
532 + FN(float, float, vllm::Fp8KVCacheDataType::kAuto); \
533 + } else if (SRC_DTYPE == at::ScalarType::Half) { \
534 + FN(uint16_t, uint16_t, vllm::Fp8KVCacheDataType::kAuto); \
535 + } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
536 + FN(__nv_bfloat16, __nv_bfloat16, vllm::Fp8KVCacheDataType::kAuto); \
537 + } else { \
538 + TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
539 + } \
540 + } else { \
541 + if (KV_DTYPE == "fp8" || KV_DTYPE == "fp8_e4m3") { \
542 + if (SRC_DTYPE == at::ScalarType::Float) { \
543 + FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
544 + } else if (SRC_DTYPE == at::ScalarType::Half) { \
545 + FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
546 + } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
547 + FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E4M3); \
548 + } else { \
549 + TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
550 + } \
551 + } else if (KV_DTYPE == "fp8_e5m2") { \
552 + if (SRC_DTYPE == at::ScalarType::Float) { \
553 + FN(float, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
554 + } else if (SRC_DTYPE == at::ScalarType::Half) { \
555 + FN(uint16_t, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
556 + } else if (SRC_DTYPE == at::ScalarType::BFloat16) { \
557 + FN(__nv_bfloat16, uint8_t, vllm::Fp8KVCacheDataType::kFp8E5M2); \
558 + } else { \
559 + TORCH_CHECK(false, "Unsupported input type of kv cache: ", SRC_DTYPE); \
560 + } \
561 + } else { \
562 + TORCH_CHECK(false, "Unsupported data type of kv cache: ", KV_DTYPE); \
563 + } \
564 + }
565 +
566 +} // namespace fp8
567 +#endif // not USE_ROCM
568 +} // namespace vllm
csrc/quantization/fp8_e5m2_kvcache/quant_utils.cuh deleted
-277
@@ -1,277 +0,0 @@
1 -#pragma once
2 -
3 -#include <assert.h>
4 -#include <stdint.h>
5 -#include <float.h>
6 -#include <type_traits>
7 -#include "../../attention/attention_dtypes.h"
8 -#include "../../attention/dtype_float32.cuh"
9 -#include "../../attention/dtype_float16.cuh"
10 -#include "../../attention/dtype_bfloat16.cuh"
11 -
12 -
13 -namespace vllm {
14 -#ifdef ENABLE_FP8_E5M2
15 -namespace fp8_e5m2_unscaled {
16 -
17 -template<typename Tout, typename Tin>
18 -__inline__ __device__ Tout vec_conversion(const Tin& x)
19 -{
20 - return x;
21 -}
22 -
23 -// fp8 -> half
24 -template<>
25 -__inline__ __device__ uint16_t vec_conversion<uint16_t, uint8_t>(const uint8_t& a)
26 -{
27 - __half_raw res = __nv_cvt_fp8_to_halfraw(a, __NV_E5M2);
28 - return res.x;
29 -}
30 -
31 -// fp8x2 -> half2
32 -template<>
33 -__inline__ __device__ uint32_t vec_conversion<uint32_t, uint16_t>(const uint16_t& a)
34 -{
35 - union {
36 - uint16_t u16[2];
37 - uint32_t u32;
38 - } tmp;
39 - __half2_raw res = __nv_cvt_fp8x2_to_halfraw2(a, __NV_E5M2);
40 - tmp.u16[0] = res.x;
41 - tmp.u16[1] = res.y;
42 - return tmp.u32;
43 -}
44 -
45 -// fp8x4 -> half2x2
46 -template<>
47 -__inline__ __device__ uint2 vec_conversion<uint2, uint32_t>(const uint32_t& a)
48 -{
49 - union {
50 - uint2 u32x2;
51 - uint32_t u32[2];
52 - } tmp;
53 - tmp.u32[0] = vec_conversion<uint32_t, uint16_t>((uint16_t)a);
54 - tmp.u32[1] = vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U));
55 - return tmp.u32x2;
56 -}
57 -
58 -// fp8x8 -> half2x4
59 -template<>
60 -__inline__ __device__ uint4 vec_conversion<uint4, uint2>(const uint2& a)
61 -{
62 - union {
63 - uint4 u64x2;
64 - uint2 u64[2];
65 - } tmp;
66 - tmp.u64[0] = vec_conversion<uint2, uint32_t>(a.x);
67 - tmp.u64[1] = vec_conversion<uint2, uint32_t>(a.y);
68 - return tmp.u64x2;
69 -}
70 -
71 -// fp8 -> __nv_bfloat16
72 -template<>
73 -__inline__ __device__ __nv_bfloat16 vec_conversion<__nv_bfloat16, uint8_t>(const uint8_t& a)
74 -{
75 - // Note there is no direct convert function from fp8 to bf16.
76 - // fp8 -> half
77 - __half_raw res = __nv_cvt_fp8_to_halfraw(a, __NV_E5M2);
78 - // half -> float -> bf16
79 - float tmp = half_to_float(res.x);
80 - return __float2bfloat16(tmp);
81 -}
82 -
83 -// fp8x2 -> __nv_bfloat162
84 -template<>
85 -__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, uint16_t>(const uint16_t& a)
86 -{
87 - __nv_bfloat162 res;
88 - res.x = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a);
89 - res.y = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U));
90 - return res;
91 -}
92 -
93 -// fp8x4 -> bf16_4_t
94 -template<>
95 -__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, uint32_t>(const uint32_t& a)
96 -{
97 - bf16_4_t res;
98 - res.x = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a);
99 - res.y = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U));
100 - return res;
101 -}
102 -
103 -// fp8x8 -> bf16_8_t
104 -template<>
105 -__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, uint2>(const uint2& a)
106 -{
107 - bf16_4_t tmp1, tmp2;
108 - tmp1 = vec_conversion<bf16_4_t, uint32_t>(a.x);
109 - tmp2 = vec_conversion<bf16_4_t, uint32_t>(a.y);
110 - bf16_8_t res;
111 - res.x = tmp1.x;
112 - res.y = tmp1.y;
113 - res.z = tmp2.x;
114 - res.w = tmp2.y;
115 - return res;
116 -}
117 -
118 -// fp8 -> float
119 -template<>
120 -__inline__ __device__ float vec_conversion<float, uint8_t>(const uint8_t& a)
121 -{
122 - // fp8 -> half
123 - uint16_t tmp = vec_conversion<uint16_t, uint8_t>(a);
124 - // half -> float
125 - return half_to_float(tmp);
126 -}
127 -
128 -// fp8x2 -> float2
129 -template<>
130 -__inline__ __device__ float2 vec_conversion<float2, uint16_t>(const uint16_t& a)
131 -{
132 - // fp8x2 -> half2
133 - uint32_t tmp = vec_conversion<uint32_t, uint16_t>(a);
134 - // half2 -> float2
135 - return half2_to_float2(tmp);
136 -}
137 -
138 -// fp8x4 -> float4
139 -template<>
140 -__inline__ __device__ Float4_ vec_conversion<Float4_, uint32_t>(const uint32_t& a)
141 -{
142 - Float4_ res;
143 - res.x = vec_conversion<float2, uint16_t>((uint16_t)a);
144 - res.y = vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U));
145 - return res;
146 -}
147 -
148 -// fp8x8 -> float8
149 -template<>
150 -__inline__ __device__ Float8_ vec_conversion<Float8_, uint2>(const uint2& a)
151 -{
152 - Float4_ tmp1, tmp2;
153 - tmp1 = vec_conversion<Float4_, uint32_t>(a.x);
154 - tmp2 = vec_conversion<Float4_, uint32_t>(a.y);
155 - Float8_ res;
156 - res.x = tmp1.x;
157 - res.y = tmp1.y;
158 - res.z = tmp2.x;
159 - res.w = tmp2.y;
160 - return res;
161 -}
162 -
163 -
164 -// half -> fp8
165 -template<>
166 -__inline__ __device__ uint8_t vec_conversion<uint8_t, uint16_t>(const uint16_t& a)
167 -{
168 - __half_raw tmp;
169 - tmp.x = a;
170 - __nv_fp8_storage_t res = __nv_cvt_halfraw_to_fp8(tmp, __NV_SATFINITE, __NV_E5M2);
171 - return (uint8_t)res;
172 -}
173 -
174 -// bf16 -> fp8
175 -template<>
176 -__inline__ __device__ uint8_t vec_conversion<uint8_t, __nv_bfloat16>(const __nv_bfloat16& a)
177 -{
178 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
179 - assert(false);
180 -#else
181 - __nv_fp8_storage_t res = __nv_cvt_bfloat16raw_to_fp8(__nv_bfloat16_raw(a), __NV_SATFINITE, __NV_E5M2);
182 - return (uint8_t)res;
183 -#endif
184 -}
185 -
186 -// float -> fp8
187 -template<>
188 -__inline__ __device__ uint8_t vec_conversion<uint8_t, float>(const float& a)
189 -{
190 - __nv_fp8_storage_t res = __nv_cvt_float_to_fp8(a, __NV_SATFINITE, __NV_E5M2);
191 - return (uint8_t)res;
192 -}
193 -
194 -// fp8x4 -> float4
195 -template<>
196 -__inline__ __device__ float4 vec_conversion<float4, uint32_t>(const uint32_t& a)
197 -{
198 - Float4_ tmp = vec_conversion<Float4_, uint32_t>(a);
199 - float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
200 - return res;
201 -}
202 -
203 -
204 -template<>
205 -__inline__ __device__ uint32_t vec_conversion<uint32_t, float2>(const float2& a)
206 -{
207 - union {
208 - half2 float16;
209 - uint32_t uint32;
210 - };
211 -
212 - float16 = __float22half2_rn(a);
213 - return uint32;
214 -}
215 -
216 -template<>
217 -__inline__ __device__ uint2 vec_conversion<uint2, Float4_>(const Float4_& a)
218 -{
219 - uint2 b;
220 - float2 val;
221 - val.x = a.x.x;
222 - val.y = a.x.y;
223 - b.x = vec_conversion<uint32_t, float2>(val);
224 -
225 - val.x = a.y.x;
226 - val.y = a.y.y;
227 - b.y = vec_conversion<uint32_t, float2>(val);
228 -
229 - return b;
230 -}
231 -
232 -template<>
233 -__inline__ __device__ float4 vec_conversion<float4, Float4_>(const Float4_& a)
234 -{
235 - float4 b;
236 - b.x = a.x.x;
237 - b.y = a.x.y;
238 - b.z = a.y.x;
239 - b.w = a.y.y;
240 - return b;
241 -}
242 -
243 -template<>
244 -__inline__ __device__ uint4 vec_conversion<uint4, Float8_>(const Float8_& a)
245 -{
246 - uint4 b;
247 - b.x = vec_conversion<uint32_t, float2>(a.x);
248 - b.y = vec_conversion<uint32_t, float2>(a.y);
249 - b.z = vec_conversion<uint32_t, float2>(a.z);
250 - b.w = vec_conversion<uint32_t, float2>(a.w);
251 - return b;
252 -}
253 -
254 -template<>
255 -__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, float2>(const float2 &a) {
256 - __nv_bfloat162 b;
257 - from_float(b, a);
258 - return b;
259 -}
260 -
261 -template<>
262 -__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, Float4_>(const Float4_ &a) {
263 - bf16_4_t b;
264 - from_float(b, a);
265 - return b;
266 -}
267 -
268 -template<>
269 -__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, Float8_>(const Float8_ &a) {
270 - bf16_8_t b;
271 - from_float(b, a);
272 - return b;
273 -}
274 -
275 -} // namespace fp8_e5m2_unscaled
276 -#endif // ENABLE_FP8_E5M2
277 -} // namespace vllm
tests/kernels/test_attention.py
+2 -2
@@ -236,14 +236,14 @@ def test_paged_attention(
236 dequantized_key_cache = torch.empty(size=key_cache_shape,
237 dtype=dtype,
238 device=device)
239 - ops.convert_fp8(key_cache, dequantized_key_cache)
239 + ops.convert_fp8(dequantized_key_cache, key_cache)
240 key_cache = dequantized_key_cache
241
242 value_cache_shape = value_cache.shape
243 dequantized_value_cache = torch.empty(size=value_cache_shape,
244 dtype=dtype,
245 device=device)
246 - ops.convert_fp8(value_cache, dequantized_value_cache)
246 + ops.convert_fp8(dequantized_value_cache, value_cache)
247 value_cache = dequantized_value_cache
248
249 ref_output = torch.empty_like(query)
tests/kernels/test_cache.py
+11 -16
@@ -5,8 +5,6 @@ import pytest
5 import torch
6
7 from vllm import _custom_ops as ops
8 -from vllm._C import cache_ops
9 -from vllm.utils import is_hip
8
9 COPYING_DIRECTION = [('cuda', 'cpu'), ('cuda', 'cuda'), ('cpu', 'cuda')]
10 DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -25,6 +23,8 @@ SEEDS = [0]
23 CUDA_DEVICES = [
24 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
25 ]
26 +
27 +# We assume fp8 is always enabled for testing.
28 KV_CACHE_DTYPE = ["auto", "fp8"]
29
30
@@ -124,8 +124,6 @@ def test_reshape_and_cache(
124 device: str,
125 kv_cache_dtype: str,
126 ) -> None:
127 - if not is_hip() and kv_cache_dtype == "fp8":
128 - pytest.skip() # This test is not tuned for e5m2 cuda precision
127 random.seed(seed)
128 torch.random.manual_seed(seed)
129 if torch.cuda.is_available():
@@ -149,9 +147,9 @@ def test_reshape_and_cache(
147 # Clone the KV caches.
148 if kv_cache_dtype == "fp8":
149 cloned_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
152 - ops.convert_fp8(key_cache, cloned_key_cache)
150 + ops.convert_fp8(cloned_key_cache, key_cache)
151 cloned_value_cache = torch.empty_like(value_cache, dtype=torch.float16)
154 - ops.convert_fp8(value_cache, cloned_value_cache)
152 + ops.convert_fp8(cloned_value_cache, value_cache)
153 else:
154 cloned_key_cache = key_cache.clone()
155 cloned_value_cache = value_cache.clone()
@@ -165,9 +163,9 @@ def test_reshape_and_cache(
163
164 if kv_cache_dtype == "fp8":
165 result_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
168 - ops.convert_fp8(key_cache, result_key_cache)
166 + ops.convert_fp8(result_key_cache, key_cache)
167 result_value_cache = torch.empty_like(value_cache, dtype=torch.float16)
170 - ops.convert_fp8(value_cache, result_value_cache)
168 + ops.convert_fp8(result_value_cache, value_cache)
169
170 # Run the reference implementation.
171 reshaped_key = key.reshape(num_tokens, *key_cache[0, :, :, 0, :].shape)
@@ -255,8 +253,8 @@ def test_reshape_and_cache_flash(
253 cloned_value_cache = value_cache.clone()
254
255 # Call the reshape_and_cache kernel.
258 - cache_ops.reshape_and_cache_flash(key, value, key_cache, value_cache,
259 - slot_mapping, kv_cache_dtype)
256 + ops.reshape_and_cache_flash(key, value, key_cache, value_cache,
257 + slot_mapping, kv_cache_dtype)
258
259 # Run the reference implementation.
260 block_indicies = torch.div(slot_mapping, block_size, rounding_mode='floor')
@@ -299,8 +297,6 @@ def test_swap_blocks(
297 ) -> None:
298 if kv_cache_dtype == "fp8" and "cpu" in direction:
299 pytest.skip()
302 - if not is_hip() and kv_cache_dtype == "fp8":
303 - pytest.skip() # This test is not tuned for e5m2 cuda precision
300 random.seed(seed)
301 torch.random.manual_seed(seed)
302 if torch.cuda.is_available():
@@ -348,7 +344,6 @@ def test_swap_blocks(
344 dist_value_caches[0][dst].cpu())
345
346
351 -@pytest.mark.skipif(not is_hip(), reason="FP8 conversion test requires e4m3")
347 @pytest.mark.parametrize("num_heads", NUM_HEADS)
348 @pytest.mark.parametrize("head_size", HEAD_SIZES)
349 @pytest.mark.parametrize("block_size", BLOCK_SIZES)
@@ -357,7 +352,7 @@ def test_swap_blocks(
352 @pytest.mark.parametrize("seed", SEEDS)
353 @pytest.mark.parametrize("device", CUDA_DEVICES)
354 @torch.inference_mode()
360 -def test_fp8_conversion(
355 +def test_fp8_e4m3_conversion(
356 num_heads: int,
357 head_size: int,
358 block_size: int,
@@ -377,9 +372,9 @@ def test_fp8_conversion(
372 cache.uniform_(low, high)
373
374 cache_fp8 = torch.empty_like(cache, dtype=torch.uint8)
380 - ops.convert_fp8(cache, cache_fp8)
375 + ops.convert_fp8(cache_fp8, cache)
376
377 converted_cache = torch.empty_like(cache)
383 - ops.convert_fp8(cache_fp8, converted_cache)
378 + ops.convert_fp8(converted_cache, cache_fp8)
379
380 assert torch.allclose(cache, converted_cache, atol=0.001, rtol=0.1)
vllm/_custom_ops.py
+5 -2
@@ -270,8 +270,11 @@ def swap_blocks(src: torch.Tensor, dst: torch.Tensor,
270 vllm_cache_ops.swap_blocks(src, dst, block_mapping)
271
272
273 -def convert_fp8(output: torch.Tensor, input: torch.Tensor) -> None:
274 - vllm_cache_ops.convert_fp8(output, input)
273 +def convert_fp8(output: torch.Tensor,
274 + input: torch.Tensor,
275 + scale: float = 1.0,
276 + kv_dtype: str = "fp8") -> None:
277 + vllm_cache_ops.convert_fp8(output, input, scale, kv_dtype)
278
279
280 #TODO: cuda_utils, custom_ar
vllm/utils.py
+1 -1
@@ -329,7 +329,7 @@ def _generate_random_fp8(
329 from vllm import _custom_ops as ops
330 tensor_tmp = torch.empty_like(tensor, dtype=torch.float16)
331 tensor_tmp.uniform_(low, high)
332 - ops.convert_fp8(tensor_tmp, tensor)
332 + ops.convert_fp8(tensor, tensor_tmp)
333 del tensor_tmp
334
335