| 1 | #pragma once |
| 2 | |
| 3 | #include "libtorch_stable/quantization/vectorization.cuh" |
| 4 | #include "../../utils.cuh" |
| 5 | |
| 6 | #include <cmath> |
| 7 | |
| 8 | // This header is shared between _C and _C_stable_libtorch targets. |
| 9 | // torch_utils.h provides get_device_prop(). We need to pass USE_CUDA |
| 10 | // to the .so to expose some of the shims used by torch_utils.h. For now |
| 11 | // this is only done for _C_stable_libtorch and not for _C, so we use the |
| 12 | // non stable at::cuda::getCurrentDeviceProperties for _C for now. |
| 13 | #ifdef TORCH_TARGET_VERSION |
| 14 | #include "../../../libtorch_stable/torch_utils.h" |
| 15 | #else |
| 16 | #ifdef USE_ROCM |
| 17 | #include <ATen/hip/HIPContext.h> |
| 18 | #endif |
| 19 | #endif |
| 20 | |
| 21 | #ifndef USE_ROCM |
| 22 | #include "nvidia/quant_utils.cuh" |
| 23 | #else |
| 24 | #include "amd/quant_utils.cuh" |
| 25 | #endif |
| 26 | |
| 27 | // Determines the preferred FP8 type for the current platform. |
| 28 | // Note that for CUDA this just returns true, |
| 29 | // but on ROCm it will check device props. |
| 30 | static bool is_fp8_ocp() { |
| 31 | #ifndef USE_ROCM |
| 32 | return true; |
| 33 | #else |
| 34 | #ifdef TORCH_TARGET_VERSION |
| 35 | auto* dprops = get_device_prop(); |
| 36 | #else |
| 37 | auto* dprops = at::cuda::getCurrentDeviceProperties(); |
| 38 | #endif |
| 39 | std::string device_arch = dprops->gcnArchName; |
| 40 | size_t substring = device_arch.find("gfx94"); |
| 41 | return substring == std::string::npos; |
| 42 | #endif |
| 43 | } |
| 44 | |
| 45 | namespace vllm { |
| 46 | |
| 47 | __device__ __forceinline__ float atomicMaxFloat(float* addr, float value) { |
| 48 | float old; |
| 49 | old = (value >= 0) |
| 50 | ? __int_as_float(atomicMax((int*)addr, __float_as_int(value))) |
| 51 | : __uint_as_float( |
| 52 | atomicMin((unsigned int*)addr, __float_as_uint(value))); |
| 53 | |
| 54 | return old; |
| 55 | } |
| 56 | |
| 57 | template <bool is_scale_inverted, typename fp8_type> |
| 58 | __device__ __forceinline__ fp8_type scaled_fp8_conversion(float const val, |
| 59 | float const scale) { |
| 60 | float x = 0.0f; |
| 61 | if constexpr (is_scale_inverted) { |
| 62 | x = val * scale; |
| 63 | } else { |
| 64 | x = val / scale; |
| 65 | } |
| 66 | |
| 67 | float r = |
| 68 | fmaxf(-quant_type_max_v<fp8_type>, fminf(x, quant_type_max_v<fp8_type>)); |
| 69 | #ifndef USE_ROCM |
| 70 | // Use hardware cvt instruction for fp8 on nvidia |
| 71 | // Currently only support fp8_type = c10::Float8_e4m3fn |
| 72 | return fp8::vec_conversion<fp8_type, float>(r); |
| 73 | #else |
| 74 | // Use hardware cvt instruction for fp8 on rocm |
| 75 | return fp8::cvt_c10<fp8_type>(r); |
| 76 | #endif |
| 77 | } |
| 78 | |
| 79 | } // namespace vllm |