| 1 | #include "machete_mm_launcher.cuh" |
| 2 | #include "machete_prepack_launcher.cuh" |
| 3 | #include "core/scalar_type.hpp" |
| 4 | |
| 5 | #include <torch/csrc/stable/library.h> |
| 6 | #include <torch/csrc/stable/tensor.h> |
| 7 | #include <torch/headeronly/core/ScalarType.h> |
| 8 | |
| 9 | namespace machete { |
| 10 | |
| 11 | using namespace vllm; |
| 12 | |
| 13 | std::vector<std::string> supported_schedules( |
| 14 | torch::headeronly::ScalarType a_type, int64_t b_type_id, |
| 15 | std::optional<torch::headeronly::ScalarType> maybe_group_scales_type, |
| 16 | std::optional<torch::headeronly::ScalarType> maybe_group_zeros_type, |
| 17 | std::optional<torch::headeronly::ScalarType> maybe_channel_scales_type, |
| 18 | std::optional<torch::headeronly::ScalarType> maybe_token_scales_type, |
| 19 | std::optional<torch::headeronly::ScalarType> maybe_out_type) { |
| 20 | ScalarType const b_type = ScalarType::from_id(b_type_id); |
| 21 | return supported_schedules_dispatch({ |
| 22 | .a_type = a_type, |
| 23 | .b_type = b_type, |
| 24 | .maybe_group_scales_type = maybe_group_scales_type, |
| 25 | .maybe_group_zeros_type = maybe_group_zeros_type, |
| 26 | .maybe_channel_scales_type = maybe_channel_scales_type, |
| 27 | .maybe_token_scales_type = maybe_token_scales_type, |
| 28 | .maybe_out_type = maybe_out_type, |
| 29 | }); |
| 30 | } |
| 31 | |
| 32 | torch::stable::Tensor mm( |
| 33 | torch::stable::Tensor const& A, torch::stable::Tensor const& B, |
| 34 | int64_t b_type_id, |
| 35 | std::optional<torch::headeronly::ScalarType> const& maybe_out_type, |
| 36 | std::optional<torch::stable::Tensor> const& maybe_group_scales, |
| 37 | std::optional<torch::stable::Tensor> const& maybe_group_zeros, |
| 38 | std::optional<int64_t> maybe_group_size, |
| 39 | std::optional<torch::stable::Tensor> const& maybe_channel_scales, |
| 40 | std::optional<torch::stable::Tensor> const& maybe_token_scales, |
| 41 | std::optional<std::string> maybe_schedule) { |
| 42 | ScalarType const b_type = ScalarType::from_id(b_type_id); |
| 43 | return mm_dispatch({.A = A, |
| 44 | .B = B, |
| 45 | .b_type = b_type, |
| 46 | .maybe_out_type = maybe_out_type, |
| 47 | .maybe_group_scales = maybe_group_scales, |
| 48 | .maybe_group_zeros = maybe_group_zeros, |
| 49 | .maybe_group_size = maybe_group_size, |
| 50 | .maybe_channel_scales = maybe_channel_scales, |
| 51 | .maybe_token_scales = maybe_token_scales, |
| 52 | .maybe_schedule = maybe_schedule}); |
| 53 | } |
| 54 | |
| 55 | torch::stable::Tensor prepack_B( |
| 56 | torch::stable::Tensor const& B, torch::headeronly::ScalarType const& a_type, |
| 57 | int64_t b_type_id, |
| 58 | std::optional<torch::headeronly::ScalarType> const& |
| 59 | maybe_group_scales_type) { |
| 60 | ScalarType const b_type = ScalarType::from_id(b_type_id); |
| 61 | return prepack_B_dispatch( |
| 62 | {.B = B, |
| 63 | .a_type = a_type, |
| 64 | .b_type = b_type, |
| 65 | .maybe_group_scales_type = maybe_group_scales_type}); |
| 66 | } |
| 67 | |
| 68 | STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) { |
| 69 | m.impl("machete_prepack_B", TORCH_BOX(&prepack_B)); |
| 70 | m.impl("machete_mm", TORCH_BOX(&mm)); |
| 71 | } |
| 72 | |
| 73 | STABLE_TORCH_LIBRARY_IMPL(_C, CompositeExplicitAutograd, m) { |
| 74 | m.impl("machete_supported_schedules", TORCH_BOX(&supported_schedules)); |
| 75 | } |
| 76 | |
| 77 | }; // namespace machete |