A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Perf] Use NVIDIA hardware-accelerated instruction for float to fp8_e4m3 quantization (#24757)
Signed-off-by: elvischenv <219235043+elvischenv@users.noreply.github.com>
elvischenv committed
Sep 13, 2025 at 15:16 UTC
dbeee3844cf93278e18e736ea7f177648de2d6aa
2 files changed
+22
-5
csrc/quantization/fp8/common.cuh
+6
-2
@@ -5,7 +5,9 @@
5
6
#include <cmath>
7
8
-#ifdef USE_ROCM
8
+#ifndef USE_ROCM
9
+ #include "nvidia/quant_utils.cuh"
10
+#else
11
#include "amd/quant_utils.cuh"
12
#endif
13
@@ -48,7 +50,9 @@ __device__ __forceinline__ fp8_type scaled_fp8_conversion(float const val,
50
float r =
51
fmaxf(-quant_type_max_v<fp8_type>, fminf(x, quant_type_max_v<fp8_type>));
52
#ifndef USE_ROCM
51
- return static_cast<fp8_type>(r);
53
+ // Use hardware cvt instruction for fp8 on nvidia
54
+ // Currently only support fp8_type = c10::Float8_e4m3fn
55
+ return fp8::vec_conversion<fp8_type, float>(r);
56
#else
57
// Use hardware cvt instruction for fp8 on rocm
58
return fp8::cvt_c10<fp8_type>(r);
csrc/quantization/fp8/nvidia/quant_utils.cuh
+16
-3
@@ -12,13 +12,26 @@ namespace vllm {
12
namespace fp8 {
13
#ifdef ENABLE_FP8
14
15
- #if 0 // Disable the following code to reduce the binary size.
15
template <typename Tout, typename Tin>
17
-__inline__ __device__ Tout
18
-vec_conversion(const Tin &x, const __nv_fp8_interpretation_t fp8_type) {
16
+__inline__ __device__ Tout vec_conversion(
17
+ const Tin& x, const __nv_fp8_interpretation_t fp8_type = __NV_E4M3) {
18
return x;
19
}
20
21
+// float -> c10::Float8_e4m3fn
22
+template <>
23
+__inline__ __device__ c10::Float8_e4m3fn
24
+vec_conversion<c10::Float8_e4m3fn, float>(
25
+ const float& a, const __nv_fp8_interpretation_t fp8_type) {
26
+ #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
27
+ return static_cast<c10::Float8_e4m3fn>(a);
28
+ #else
29
+ return c10::Float8_e4m3fn(__nv_cvt_float_to_fp8(a, __NV_SATFINITE, fp8_type),
30
+ c10::Float8_e4m3fn::from_bits());
31
+ #endif
32
+}
33
+
34
+ #if 0 // Disable the following code to reduce the binary size.
35
// fp8 -> half
36
template <>
37
__inline__ __device__ uint16_t vec_conversion<uint16_t, uint8_t>(