| 1 | #pragma once |
| 2 | |
| 3 | #include "machete_mm_kernel.cuh" |
| 4 | #include "libtorch_stable/cutlass_extensions/torch_utils.hpp" |
| 5 | #include "core/scalar_type.hpp" |
| 6 | #include "libtorch_stable/torch_utils.h" |
| 7 | |
| 8 | #include <torch/csrc/stable/tensor.h> |
| 9 | #include <torch/headeronly/core/ScalarType.h> |
| 10 | |
| 11 | #include <optional> |
| 12 | #include <string> |
| 13 | #include <vector> |
| 14 | |
| 15 | namespace machete { |
| 16 | |
| 17 | struct MMArgs { |
| 18 | torch::stable::Tensor const& A; |
| 19 | torch::stable::Tensor const& B; |
| 20 | vllm::ScalarType const& b_type; |
| 21 | std::optional<torch::headeronly::ScalarType> const& maybe_out_type; |
| 22 | std::optional<torch::stable::Tensor> const& maybe_group_scales; |
| 23 | std::optional<torch::stable::Tensor> const& maybe_group_zeros; |
| 24 | std::optional<int64_t> maybe_group_size; |
| 25 | std::optional<torch::stable::Tensor> const& maybe_channel_scales; |
| 26 | std::optional<torch::stable::Tensor> const& maybe_token_scales; |
| 27 | std::optional<std::string> maybe_schedule; |
| 28 | }; |
| 29 | |
| 30 | struct SupportedSchedulesArgs { |
| 31 | torch::headeronly::ScalarType a_type; |
| 32 | vllm::ScalarType b_type; |
| 33 | std::optional<torch::headeronly::ScalarType> maybe_group_scales_type; |
| 34 | std::optional<torch::headeronly::ScalarType> maybe_group_zeros_type; |
| 35 | std::optional<torch::headeronly::ScalarType> maybe_channel_scales_type; |
| 36 | std::optional<torch::headeronly::ScalarType> maybe_token_scales_type; |
| 37 | std::optional<torch::headeronly::ScalarType> maybe_out_type; |
| 38 | }; |
| 39 | |
| 40 | torch::stable::Tensor mm_dispatch(MMArgs args); |
| 41 | |
| 42 | std::vector<std::string> supported_schedules_dispatch( |
| 43 | SupportedSchedulesArgs args); |
| 44 | |
| 45 | template <typename MacheteKernel> |
| 46 | torch::stable::Tensor run_impl(MMArgs args) { |
| 47 | const torch::stable::accelerator::DeviceGuard device_guard( |
| 48 | args.A.get_device_index()); |
| 49 | |
| 50 | auto device = args.A.device(); |
| 51 | auto stream = get_current_cuda_stream(device.index()); |
| 52 | |
| 53 | int M = args.A.size(0); |
| 54 | int N = args.B.size(1); |
| 55 | int K = args.A.size(1); |
| 56 | |
| 57 | // Allocate output |
| 58 | torch::stable::Tensor D = torch::stable::empty( |
| 59 | {M, N}, equivalent_scalar_type_v<typename MacheteKernel::ElementD>, |
| 60 | std::nullopt, device); |
| 61 | |
| 62 | auto arguments = MacheteKernel::create_arguments( |
| 63 | stream, // |
| 64 | args.A, args.B, D, args.maybe_group_scales, args.maybe_group_zeros, |
| 65 | args.maybe_group_size, args.maybe_channel_scales, |
| 66 | args.maybe_token_scales); |
| 67 | STD_TORCH_CHECK(MacheteKernel::can_implement(arguments), |
| 68 | "Machete kernel cannot be run with these arguments"); |
| 69 | |
| 70 | size_t workspace_size = MacheteKernel::get_workspace_size(arguments); |
| 71 | torch::stable::Tensor workspace = |
| 72 | torch::stable::empty(workspace_size, torch::headeronly::ScalarType::Byte, |
| 73 | std::nullopt, device); |
| 74 | |
| 75 | MacheteKernel::run(arguments, workspace.mutable_data_ptr(), stream); |
| 76 | |
| 77 | return D; |
| 78 | }; |
| 79 | |
| 80 | }; // namespace machete |