| 1 | #pragma once |
| 2 | |
| 3 | #include <torch/all.h> |
| 4 | |
| 5 | torch::Tensor LLMM1(at::Tensor& in_a, at::Tensor& in_b, |
| 6 | const int64_t rows_per_block); |
| 7 | |
| 8 | torch::Tensor wvSplitK(const at::Tensor& in_a, const at::Tensor& in_b, |
| 9 | const std::optional<at::Tensor>& in_bias, |
| 10 | const int64_t CuCount); |
| 11 | |
| 12 | torch::Tensor wvSplitK_int4_g(const at::Tensor& in_a, const at::Tensor& in_b, |
| 13 | const at::Tensor& in_scale, |
| 14 | const std::optional<at::Tensor>& in_zero_points, |
| 15 | const std::optional<at::Tensor>& in_bias, |
| 16 | const int64_t CuCount, const int64_t group_size); |
| 17 | |
| 18 | torch::Tensor wvSplitKrc(const at::Tensor& in_a, const at::Tensor& in_b, |
| 19 | const std::optional<at::Tensor>& in_bias, |
| 20 | const int64_t CuCount); |
| 21 | |
| 22 | void wvSplitKQ(const at::Tensor& in_a, const at::Tensor& in_b, |
| 23 | const std::optional<at::Tensor>& in_bias, at::Tensor& out_c, |
| 24 | const at::Tensor& scale_a, const at::Tensor& scale_b, |
| 25 | const int64_t CuCount); |
| 26 | |
| 27 | torch::Tensor gptq_gemm_rdna3(torch::Tensor a, torch::Tensor b_q_weight, |
| 28 | torch::Tensor b_qzeros, torch::Tensor b_scales, |
| 29 | bool use_v2_format); |
| 30 | |
| 31 | torch::Tensor gptq_gemm_rdna3_wmma(torch::Tensor a, torch::Tensor b_q_weight, |
| 32 | torch::Tensor b_qzeros, |
| 33 | torch::Tensor b_scales, bool use_v2_format); |
| 34 | |
| 35 | void moe_gptq_gemm_rdna3(torch::Tensor a, torch::Tensor c, |
| 36 | torch::Tensor b_q_weight, torch::Tensor b_scales, |
| 37 | torch::Tensor b_qzeros, torch::Tensor topk_weights, |
| 38 | torch::Tensor sorted_token_ids, |
| 39 | torch::Tensor expert_ids, |
| 40 | torch::Tensor num_tokens_post_padded, int64_t top_k, |
| 41 | int64_t block_size_m, bool mul_topk_weight, |
| 42 | int64_t output_topk); |
| 43 | |
| 44 | void paged_attention( |
| 45 | torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits, |
| 46 | torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache, |
| 47 | torch::Tensor& value_cache, int64_t num_kv_heads, double scale, |
| 48 | torch::Tensor& block_tables, torch::Tensor& seq_lens, |
| 49 | const std::optional<torch::Tensor>& query_start_loc, int64_t block_size, |
| 50 | int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes, |
| 51 | const std::string& kv_cache_dtype, torch::Tensor& k_scale, |
| 52 | torch::Tensor& v_scale, const std::optional<torch::Tensor>& fp8_out_scale, |
| 53 | const std::string& mfma_type); |