| 1 | #pragma once |
| 2 | |
| 3 | /** |
| 4 | * __device__ helper functions to deal with float -> quant datatype conversion |
| 5 | */ |
| 6 | |
| 7 | #include "libtorch_stable/quantization/vectorization.cuh" |
| 8 | // TODO(luka/varun):refactor common.cuh to use this file instead |
| 9 | #include "../../../quantization/w8a8/fp8/common.cuh" |
| 10 | |
| 11 | namespace vllm { |
| 12 | |
| 13 | // TODO(luka/varun): combine into common utilities for int8 |
| 14 | // (with int8_quant_kernels.cu) |
| 15 | static __device__ __forceinline__ int8_t float_to_int8_rn(float const x) { |
| 16 | #ifdef USE_ROCM |
| 17 | static const float i8_min = |
| 18 | static_cast<float>(std::numeric_limits<int8_t>::min()); |
| 19 | static const float i8_max = |
| 20 | static_cast<float>(std::numeric_limits<int8_t>::max()); |
| 21 | // round |
| 22 | float dst = std::nearbyint(x); |
| 23 | // saturate |
| 24 | |
| 25 | // See https://github.com/pytorch/pytorch/issues/127666 |
| 26 | // See https://github.com/llvm/llvm-project/issues/95183 |
| 27 | // hip-clang std::clamp __glibcxx_assert_fail host function when building on |
| 28 | // Arch/gcc14. The following replaces std::clamp usage with similar logic |
| 29 | // dst = std::clamp(dst, i8_min, i8_max); |
| 30 | dst = (dst < i8_min) ? i8_min : (dst > i8_max) ? i8_max : dst; |
| 31 | return static_cast<int8_t>(dst); |
| 32 | #else |
| 33 | // CUDA path |
| 34 | uint32_t dst; |
| 35 | asm volatile("cvt.rni.sat.s8.f32 %0, %1;" : "=r"(dst) : "f"(x)); |
| 36 | return reinterpret_cast<const int8_t&>(dst); |
| 37 | #endif |
| 38 | } |
| 39 | |
| 40 | template <typename fp8_type> |
| 41 | static __device__ __forceinline__ fp8_type float_to_fp8(float const x) { |
| 42 | float const r = |
| 43 | fmax(-quant_type_max_v<fp8_type>, fmin(x, quant_type_max_v<fp8_type>)); |
| 44 | return static_cast<fp8_type>(r); |
| 45 | } |
| 46 | |
| 47 | template <typename quant_type_t, bool is_scale_inverted, typename enable = void> |
| 48 | struct ScaledQuant; |
| 49 | |
| 50 | template <typename quant_type_t, bool is_scale_inverted> |
| 51 | struct ScaledQuant< |
| 52 | quant_type_t, is_scale_inverted, |
| 53 | typename std::enable_if_t<std::is_same_v<quant_type_t, int8_t>>> { |
| 54 | static __device__ __forceinline__ quant_type_t quant_fn(float const x, |
| 55 | float const scale) { |
| 56 | if constexpr (is_scale_inverted) { |
| 57 | return float_to_int8_rn(x * scale); |
| 58 | } else { |
| 59 | return float_to_int8_rn(x / scale); |
| 60 | } |
| 61 | } |
| 62 | }; |
| 63 | |
| 64 | template <typename quant_type_t, bool is_scale_inverted> |
| 65 | struct ScaledQuant<quant_type_t, is_scale_inverted, |
| 66 | typename std::enable_if_t< |
| 67 | std::is_same_v<quant_type_t, c10::Float8_e4m3fn> || |
| 68 | std::is_same_v<quant_type_t, c10::Float8_e4m3fnuz>>> { |
| 69 | static __device__ __forceinline__ quant_type_t quant_fn(float const x, |
| 70 | float const scale) { |
| 71 | if constexpr (is_scale_inverted) { |
| 72 | return float_to_fp8<quant_type_t>(x * scale); |
| 73 | } else { |
| 74 | return float_to_fp8<quant_type_t>(x / scale); |
| 75 | } |
| 76 | } |
| 77 | }; |
| 78 | |
| 79 | } // namespace vllm |