| 1 | #pragma once |
| 2 | /** |
| 3 | * __device__ datatypes vectorized by 4 |
| 4 | */ |
| 5 | |
| 6 | // Include both AMD and NVIDIA fp8 types to avoid circular import |
| 7 | #include <torch/headeronly/util/Float8_e4m3fnuz.h> |
| 8 | #include <torch/headeronly/util/Float8_e4m3fn.h> |
| 9 | |
| 10 | namespace vllm { |
| 11 | |
| 12 | // Vectorization containers |
| 13 | template <typename scalar_t, size_t vec_size> |
| 14 | struct __align__(vec_size * sizeof(scalar_t)) vec_n_t { |
| 15 | scalar_t val[vec_size]; |
| 16 | }; |
| 17 | |
| 18 | template <typename quant_type_t, size_t vec_size> |
| 19 | struct __align__(vec_size * sizeof(quant_type_t)) q8_n_t { |
| 20 | static_assert(std::is_same_v<quant_type_t, int8_t> || |
| 21 | std::is_same_v<quant_type_t, c10::Float8_e4m3fn> || |
| 22 | std::is_same_v<quant_type_t, c10::Float8_e4m3fnuz>); |
| 23 | quant_type_t val[vec_size]; |
| 24 | }; |
| 25 | |
| 26 | template <typename scalar_t> |
| 27 | using vec4_t = vec_n_t<scalar_t, 4>; |
| 28 | template <typename quant_type_t> |
| 29 | using q8x4_t = q8_n_t<quant_type_t, 4>; |
| 30 | |
| 31 | } // namespace vllm |